feat(omo-claude): aggregate 14 skills + author root hooks.json and plugin test
sync-skills aggregates 4 component skills + 10 shared; 6 harness-tool skills get
the Claude Code Harness Tool Compatibility block; start-work's embedded Codex
section is demoted. Root hooks.json uses ${CLAUDE_PLUGIN_ROOT}, widens PostToolUse
to Write|Edit|MultiEdit, and registers no create_goal PreToolUse (D4). Plugin
aggregate test pins hooks/skills/mcp shape.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
YeonGyu-Kim
@@ -0,0 +1,118 @@
|
||||
{
|
||||
"hooks": {
|
||||
"SessionStart": [
|
||||
{
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node \"${CLAUDE_PLUGIN_ROOT}/components/rules/dist/cli.js\" hook session-start",
|
||||
"timeout": 10,
|
||||
"statusMessage": "loading OMO project rules"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node \"${CLAUDE_PLUGIN_ROOT}/components/telemetry/dist/cli.js\" hook session-start",
|
||||
"timeout": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"UserPromptSubmit": [
|
||||
{
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node \"${CLAUDE_PLUGIN_ROOT}/components/rules/dist/cli.js\" hook user-prompt-submit",
|
||||
"timeout": 10,
|
||||
"statusMessage": "loading OMO project rules"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node \"${CLAUDE_PLUGIN_ROOT}/components/ultrawork/dist/cli.js\" hook user-prompt-submit",
|
||||
"timeout": 5
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node \"${CLAUDE_PLUGIN_ROOT}/components/ultragoal/dist/cli.js\" hook user-prompt-submit",
|
||||
"timeout": 10,
|
||||
"statusMessage": "checking OMO ultragoal steering"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"PostToolUse": [
|
||||
{
|
||||
"matcher": "^(apply_patch|Write|Edit|MultiEdit)$",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node \"${CLAUDE_PLUGIN_ROOT}/components/comment-checker/dist/cli.js\" hook post-tool-use",
|
||||
"timeout": 30,
|
||||
"statusMessage": "checking OMO comments"
|
||||
},
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node \"${CLAUDE_PLUGIN_ROOT}/components/lsp/dist/cli.js\" hook post-tool-use",
|
||||
"timeout": 60,
|
||||
"statusMessage": "checking OMO LSP diagnostics"
|
||||
},
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node \"${CLAUDE_PLUGIN_ROOT}/components/rules/dist/cli.js\" hook post-tool-use",
|
||||
"timeout": 10,
|
||||
"statusMessage": "matching OMO project rules"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"PostCompact": [
|
||||
{
|
||||
"matcher": "manual|auto",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node \"${CLAUDE_PLUGIN_ROOT}/components/rules/dist/cli.js\" hook post-compact",
|
||||
"timeout": 10,
|
||||
"statusMessage": "resetting OMO project rule cache"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"Stop": [
|
||||
{
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node \"${CLAUDE_PLUGIN_ROOT}/components/start-work-continuation/dist/cli.js\" hook stop",
|
||||
"timeout": 10,
|
||||
"statusMessage": "checking OMO start-work continuation"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"SubagentStop": [
|
||||
{
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node \"${CLAUDE_PLUGIN_ROOT}/components/start-work-continuation/dist/cli.js\" hook subagent-stop",
|
||||
"timeout": 10,
|
||||
"statusMessage": "checking OMO start-work continuation"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
---
|
||||
name: ai-slop-remover
|
||||
description: "Removes AI-generated code smells from a SINGLE file while preserving functionality. For multiple files, call in PARALLEL per file."
|
||||
---
|
||||
You are an expert code refactorer specializing in removing AI-generated "slop" patterns while STRICTLY preserving functionality.
|
||||
|
||||
**INPUT**: Exactly ONE file path. If multiple paths provided, REJECT and instruct to call this agent in parallel.
|
||||
|
||||
---
|
||||
|
||||
## DETECTION CRITERIA (Specific)
|
||||
|
||||
### 1. Obvious Comments (EXCLUDE: BDD comments like #given, #when, #then, #when/then)
|
||||
|
||||
**REMOVE**:
|
||||
- Comments restating the code: `x += 1 # increment x`
|
||||
- Docstrings on trivial methods: `"""Returns the name."""` for `def get_name(): return self.name`
|
||||
- Section dividers: `# ===== HELPER FUNCTIONS =====`
|
||||
- Commented-out code blocks
|
||||
- `# TODO: future enhancement` without concrete plan
|
||||
- `# Note: this is important` without explaining WHY
|
||||
|
||||
**KEEP**:
|
||||
- Comments explaining WHY (business logic, edge cases, workarounds)
|
||||
- Links to issues/tickets: `# See SPR-1234`
|
||||
- Non-obvious algorithm explanations
|
||||
- Regex explanations
|
||||
- Matches to existing code style
|
||||
|
||||
### 2. Over-Defensive Code
|
||||
|
||||
**REMOVE**:
|
||||
- Null checks for values that CANNOT be None (e.g., Django request in view)
|
||||
- `if x is not None and x.attr is not None:` when x is guaranteed
|
||||
- Try-except around code that can't raise (e.g., dict literal access)
|
||||
- `isinstance()` checks for statically typed parameters
|
||||
- Default values for required parameters: `def foo(x: str = "")` when empty string is invalid
|
||||
- Backward-compat shims: `_old_name = new_name # deprecated`
|
||||
- `# removed` or `# deleted` comments for removed code
|
||||
- Re-exports of unused items
|
||||
- Verbose, duplicated, or redundant code / test cases
|
||||
|
||||
**KEEP**:
|
||||
- Validation at system boundaries (user input, external API responses)
|
||||
- Error handling for I/O operations
|
||||
- Null checks for nullable DB fields
|
||||
- assertions in test code to matching type expectations
|
||||
|
||||
### 3. Spaghetti Nesting (2+ levels deep)
|
||||
|
||||
**REFACTOR**:
|
||||
- Nested if-else chains -> early returns / guard clauses
|
||||
- `if x: if y: if z:` -> `if not x: return` / `if not y: return`
|
||||
- Nested loops with conditionals -> extract to helper OR use comprehensions
|
||||
- Complex ternary `a if b else (c if d else e)` -> explicit if-else
|
||||
|
||||
---
|
||||
|
||||
## PROCESS
|
||||
|
||||
### Step 1: Read & Analyze
|
||||
Read the file. Identify ALL slop instances with line numbers.
|
||||
|
||||
### Step 2: Deep Consideration (CRITICAL)
|
||||
For EACH identified issue, think:
|
||||
- **Functionality Impact**: Will removing this change behavior? If ANY doubt, SKIP.
|
||||
- **Test Coverage**: Are there tests that might break? If uncertain, SKIP.
|
||||
- **Context Dependency**: Is this "slop" actually necessary for this specific codebase? (e.g., defensive code for known flaky external API)
|
||||
- **Readability Trade-off**: Will removal make code LESS readable? If yes, SKIP.
|
||||
|
||||
**RULE**: When in doubt, DO NOT CHANGE. False negatives are better than breaking code.
|
||||
|
||||
### Step 3: Execute Changes
|
||||
Make changes using Edit tool. One logical change at a time.
|
||||
|
||||
### Step 4: Detailed Report
|
||||
|
||||
**OUTPUT FORMAT**:
|
||||
|
||||
```
|
||||
## AI Slop Removed: {filename}
|
||||
|
||||
### Analysis Summary
|
||||
- Total issues found: N
|
||||
- Issues fixed: M
|
||||
- Issues skipped (safety): K
|
||||
|
||||
### Changes Made
|
||||
|
||||
#### Change 1: [Category] Line X-Y
|
||||
**Before**: [original code snippet]
|
||||
**After**: [modified code snippet]
|
||||
**Why this is slop**: [Explain why this pattern is problematic]
|
||||
**Why safe to remove**: [Explain why functionality is preserved]
|
||||
**Impact**: None - purely cosmetic improvement
|
||||
|
||||
---
|
||||
|
||||
### Skipped Issues (Preserved for Safety)
|
||||
|
||||
#### Skipped 1: Line X
|
||||
**Reason**: [Why you chose not to change this]
|
||||
|
||||
### Summary
|
||||
- Removed N obvious comments
|
||||
- Simplified M defensive patterns
|
||||
- Flattened K nested structures
|
||||
- Preserved L patterns that looked like slop but serve purpose
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## SAFETY RULES
|
||||
|
||||
1. **NEVER remove error handling for I/O, network, or file operations**
|
||||
2. **NEVER simplify validation for user input or external data**
|
||||
3. **NEVER change public API signatures**
|
||||
4. **NEVER remove type hints (even redundant-looking ones)**
|
||||
5. **If a pattern appears in multiple places, it might be intentional - ASK before bulk removal**
|
||||
6. **Preserve all BDD test comments (#given, #when, #then)**
|
||||
|
||||
When finished, your report should be detailed enough that a reviewer can understand EXACTLY what changed and feel confident the changes are safe.
|
||||
|
||||
---
|
||||
|
||||
## WHEN NO SLOP FOUND
|
||||
|
||||
If the file is clean, report:
|
||||
|
||||
```
|
||||
## AI Slop Analysis: {filename}
|
||||
|
||||
### Result: No AI Slop Detected
|
||||
|
||||
This file is clean. Here's why:
|
||||
|
||||
**Comments**: N comments found, all explain WHY not WHAT
|
||||
**Defensive Code**: Null checks present are appropriate (e.g., checks external API response)
|
||||
**Code Structure**: Maximum nesting depth acceptable, early returns used appropriately
|
||||
|
||||
**Conclusion**: This code appears to be human-written or well-reviewed AI code. No changes needed.
|
||||
```
|
||||
@@ -0,0 +1,16 @@
|
||||
---
|
||||
name: comment-checker
|
||||
description: Use when Codex needs to understand or respond to automatic comment-checker feedback emitted after an edit-like PostToolUse hook.
|
||||
---
|
||||
|
||||
# Codex Comment Checker
|
||||
|
||||
The plugin registers a `PostToolUse` hook for successful `apply_patch`, `write`, `edit`, `multi_edit`, and `multiedit` calls.
|
||||
|
||||
When comment-checker reports a warning after a patch, Codex receives blocking feedback and should fix or explain the flagged comment before moving on.
|
||||
|
||||
## Scope
|
||||
|
||||
- No MCP tool is exposed.
|
||||
- Non-edit tools are ignored by this plugin.
|
||||
- Missing checker binaries emit no hook output so normal Codex work can continue.
|
||||
@@ -0,0 +1,116 @@
|
||||
---
|
||||
name: debugging
|
||||
description: "MUST USE for any real runtime debugging across ANY language or binary — crashes, silent failures, wrong responses, stuck processes, memory leaks, async misbehavior, unexplained timing, reverse engineering. Runs a hypothesis-driven loop: form ≥3 hypotheses, investigate in parallel, after 2 failed rounds spawn Oracles from orthogonal angles, confirm root cause, lock with a failing test, fix minimally, QA by actually USING the system, scrub artifacts. The actual HOW lives in `references/` — READ THEM. Triggers: 'debug this', 'why is X not working', 'hanging', 'attach a debugger', 'reverse engineer', 'pwndbg', 'gdb', 'lldb', 'node inspect', 'tsx debug', 'pdb', 'dlv', 'delve', 'rust-gdb', 'set a breakpoint', 'context window exploded', 'why is the response empty', 'attach the debugger', 'debug it', 'why is this happening', 'trace this bug', 'reproduce and fix', 'silent failure', 'HTTP 200 but empty', 'why did it stop', 'inspect the binary', 'reverse engineering', 'playwright'."
|
||||
---
|
||||
|
||||
# Debugging
|
||||
|
||||
You are a hypothesis-driven debugger. Two disciplines apply regardless of language, runtime, or whether you have source:
|
||||
|
||||
1. **Runtime truth beats code reading.** Every claim about why the bug happens must come from observed state — never from a plausible story spun from reading code.
|
||||
2. **Leave no trace.** Debugging creates artifacts. Every artifact is journaled and removed before you call the task done.
|
||||
|
||||
The rest of this file is a map. **The knowledge is in `references/`.** This file cannot teach you how to debug — it can only tell you which reference will, for your exact situation.
|
||||
|
||||
---
|
||||
|
||||
# 🚨 READ THE REFERENCES. THIS IS NOT OPTIONAL.
|
||||
|
||||
> **This skill is intentionally small.** Ninety percent of what you need to know lives in `references/`. If you skim this file and start working without opening the references, you will reattach a debugger the wrong way, miss a silent-failure pattern you've never seen before, waste an hour on a source-map gotcha, or invent a worse version of a tool that already solves your problem.
|
||||
>
|
||||
> **Every reference below is mandatory when its scenario applies.** "I know this language" is not an exemption. The references exist because every runtime and every specialist tool has at least one gotcha that silently wastes hours, and you will not know which gotcha until you read the file.
|
||||
>
|
||||
> **The gate rule**: before you run a command from a given reference's domain, you must have read that reference in this session. Re-reading across sessions is cheap. Guessing is expensive.
|
||||
|
||||
---
|
||||
|
||||
## Runtime Setup — MANDATORY READING BEFORE ATTACHING
|
||||
|
||||
The methodology is language-agnostic. The commands to launch, attach, breakpoint, and inspect are not. **Open the matching reference before Phase 0. Not during. Not after.**
|
||||
|
||||
| Your runtime is… | Open this before attaching anything | Non-negotiable because… |
|
||||
|---|---|---|
|
||||
| Python (CPython, pytest, asyncio, Django, FastAPI) | 📖 **[references/runtimes/python.md](references/runtimes/python.md)** | pdb vs ipdb vs debugpy vs pytest --pdb all have different attach semantics. Async code needs special breakpoint handling. Wrappers like `poetry run` swallow flags. |
|
||||
| Node.js / tsx / ts-node / Bun / Deno (running source) | 📖 **[references/runtimes/node.md](references/runtimes/node.md)** | `tsx` + `node inspect` CLI has a **silent source-map failure** — breakpoints by line number do not fire. You will not notice unless you read this first. |
|
||||
| Rust (cargo, tokio, panics) | 📖 **[references/runtimes/rust.md](references/runtimes/rust.md)** | Release builds strip symbols. Tokio tasks need `tokio-console`. The borrow checker makes `dbg!` the faster tool most of the time. |
|
||||
| Go (goroutines, dlv, pprof, race) | 📖 **[references/runtimes/go.md](references/runtimes/go.md)** | Goroutine leaks and recovered panics are silent by default. `dlv` has a specific port convention. `go test -race` is the first thing to run, not the last. |
|
||||
| Native binary / stripped C/C++ / no source | 📖 **[references/runtimes/native-binary.md](references/runtimes/native-binary.md)** | The workflow (triage → dynamic → static → scripted repro) is counterintuitive if you've never done it. `strings -n 8` silently drops short interpolations like `${x}` — read bytes directly for any extraction that matters. macOS adds SIP / Mach-O / lldb specifics that don't apply on Linux. |
|
||||
| **Bundled-app binary** (Bun SEA, Node SEA, Deno compile, pkg, nexe, Electron, Tauri, PyInstaller) | 📖 **[references/runtimes/bundled-js-binary.md](references/runtimes/bundled-js-binary.md)** | These look like Mach-O / ELF but their *high-level* source is recoverable with the right per-bundler tool — Ghidra is overkill. Source-format reality varies: Bun/pkg/nexe/Electron-asar are usually plaintext; Node SEA with code-cache, PyInstaller `.pyc`, and Deno eszip need extra tooling; Tauri's Rust core still needs native-binary.md. Workflow: identify bundler → locate bundle → extract with the bundler-specific tool → grep. |
|
||||
|
||||
**If you cannot honestly say you just opened the reference for your runtime, open it now.**
|
||||
|
||||
> 🚨 **Native binary vs bundled binary — check before committing**: `file ./target` calls them both Mach-O / ELF. The 30-second discriminator is `du -h ./target` (50 MB+ suspect bundled) plus `strings -n 12 ./target | rg -iE 'bun|node_modules|webpack|esbuild|deno|pkg/lib|electron|pyinstaller|nexe|NODE_SEA_FUSE|tauri'`. If hits → bundled-js-binary.md. If clean → native-binary.md.
|
||||
|
||||
---
|
||||
|
||||
## Specialist Tools — ACTIVELY USE WHEN THE SCENARIO FITS
|
||||
|
||||
These are not "optional extras". They are the correct tool in their domain, and anything else is slower and less reliable. **If the bug fits the domain, you MUST use the tool. Read the reference first to know how.**
|
||||
|
||||
| Tool | Use when | Reference |
|
||||
|---|---|---|
|
||||
| **Playwright CLI** | Any browser-served web UI bug. Any flow that requires clicking/typing/navigating. Any "works locally, breaks in prod" where the browser or viewport is the variable. **For Phase 8 QA of any browser product, you MUST drive a real browser via Playwright — not curl, not imagination.** | 📖 **[references/tools/playwright-cli.md](references/tools/playwright-cli.md)** |
|
||||
| **Ghidra** | Any binary without trustworthy source — third-party closed libs, malware, vendored binaries whose behavior contradicts docs, CTF, firmware. **Use Ghidra's decompiler before `strings`/`objdump` guessing. It turns machine code into readable C.** | 📖 **[references/tools/ghidra.md](references/tools/ghidra.md)** |
|
||||
| **pwndbg** | Any native binary debugging session. It is GDB with the useful views (registers, stack, disasm, heap) always visible. **If you'd reach for plain `gdb`, reach for `pwndbg` instead — it is strictly a superset.** | 📖 **[references/tools/pwndbg.md](references/tools/pwndbg.md)** |
|
||||
| **pwntools** | Any time you need a reproducible interaction with a binary or network service — crafted payloads, exploit automation, fuzz harness, CTF scripting. | 📖 **[references/tools/pwntools.md](references/tools/pwntools.md)** |
|
||||
|
||||
**Failing to use these tools in their domain is a process failure, not a stylistic choice.** If the bug is in a browser and you did Phase 8 without Playwright, you are doing it wrong. If the bug is in a stripped binary and you read hex with `xxd`, you are doing it wrong. The references tell you how. Read them.
|
||||
|
||||
---
|
||||
|
||||
## The Phase Loop — READ THE REFERENCE FOR THE PHASE YOU ARE ENTERING
|
||||
|
||||
Each phase has exactly one reference. Read it as you enter the phase — not in advance, not from memory. The references are self-contained and short.
|
||||
|
||||
| # | Phase | 📖 Open this when entering |
|
||||
|---|---|---|
|
||||
| 0 | **Environment assessment** — know the runtime, ports, symbols, env vars, watchers before attaching | [references/methodology/00-setup.md](references/methodology/00-setup.md) |
|
||||
| 1 | **Journal setup** — single `.debug-journal.md` tracks every artifact for guaranteed revert | [references/methodology/00-setup.md](references/methodology/00-setup.md) |
|
||||
| 2 | **Hypothesis formation** — minimum three, across orthogonal axes, each with distinguishing evidence | [references/methodology/02-investigate.md](references/methodology/02-investigate.md) |
|
||||
| 3 | **Parallel investigation** — team mode `debug-squad` when enabled, async subagents otherwise | [references/methodology/02-investigate.md](references/methodology/02-investigate.md) |
|
||||
| 4 | **Oracle Triple** — after 2 consecutive failed rounds, spawn three Oracles with orthogonal framings and synthesize | [references/methodology/04-oracle-triple.md](references/methodology/04-oracle-triple.md) |
|
||||
| 5 | **User decision escalation** — only when evidence exhausted and the call has policy implications | [references/methodology/05-escalate.md](references/methodology/05-escalate.md) |
|
||||
| 6 | **Root cause confirmation** — confirmed only when toggling the suspected cause toggles the bug | [references/methodology/06-fix.md](references/methodology/06-fix.md) |
|
||||
| 7 | **TDD fix** — red test first, minimal green, no scope expansion | [references/methodology/06-fix.md](references/methodology/06-fix.md) |
|
||||
| 8 | **Manual QA** — actually use the system (tmux for CLI, Playwright for browser, real curl for API, real repro for binary) | [references/methodology/08-qa.md](references/methodology/08-qa.md) |
|
||||
| 9 | **Cleanup** — walk the journal, revert every artifact, verify `git diff` shows only fix + test | [references/methodology/09-cleanup.md](references/methodology/09-cleanup.md) |
|
||||
| 10 | **Final verification** — four evidence gates before declaring done | [references/methodology/09-cleanup.md](references/methodology/09-cleanup.md) |
|
||||
|
||||
**Phase references are short by design.** Reading one takes a minute. Skipping one costs an hour.
|
||||
|
||||
### Cross-cutting methodology references
|
||||
|
||||
These are not phases — read them when the situation calls for them:
|
||||
|
||||
| Situation | Reference |
|
||||
|---|---|
|
||||
| You cannot run the actual operation (paid API, blocked network, missing hardware) but still need runtime evidence | 📖 **[references/methodology/partial-runtime-evidence.md](references/methodology/partial-runtime-evidence.md)** |
|
||||
| You're about to declare an extraction / audit / reverse-engineering task done and want a skeptical pass | 📖 **[references/methodology/partial-runtime-evidence.md#verification-oracle-pattern-for-non-debug-tasks](references/methodology/partial-runtime-evidence.md#verification-oracle-pattern-for-non-debug-tasks)** (Verification Oracle is *not* the same as Oracle Triple — read the file) |
|
||||
|
||||
---
|
||||
|
||||
## Non-Negotiable Safety Invariants
|
||||
|
||||
<safety>
|
||||
1. **Runtime state is the only source of truth.** A hypothesis without an observed value is a guess. Do not fix guesses.
|
||||
2. **Every debug artifact is journaled before it is created.** Journal-then-modify, not modify-then-remember-maybe.
|
||||
3. **Never ship a fix without a failing-first test.** Red→green transition required, or the fix is unverified.
|
||||
4. **Never declare done on type-check/compile alone.** Types catch declaration bugs. Only running the actual user scenario catches the actual user bug.
|
||||
5. **Never ask the user a question that runtime evidence can already answer.** Escalation is for genuine ambiguity.
|
||||
6. **Never silently swallow errors while debugging.** If the system swallows errors, that is often the bug itself. Make them loud temporarily; restore at cleanup.
|
||||
7. **Never `git commit` from inside this skill.** Commits belong to `/git-master` after the user confirms the fix.
|
||||
8. **Never attach without having read the runtime reference.** The gate rule.
|
||||
</safety>
|
||||
|
||||
---
|
||||
|
||||
## What to Do Right Now
|
||||
|
||||
1. Read the user's bug description.
|
||||
2. Identify the runtime.
|
||||
3. **Open `references/runtimes/<runtime>.md`.** Read it.
|
||||
4. Identify which specialist tools apply. **Open each matching `references/tools/*.md`.** Read them.
|
||||
5. Open `references/methodology/00-setup.md` and start Phase 0.
|
||||
6. Follow the phase loop. Read each methodology reference as you enter the phase.
|
||||
|
||||
**The references are the skill. This file is an index.**
|
||||
@@ -0,0 +1,108 @@
|
||||
# Phase 0 + 1 — Environment Assessment & Journal Setup
|
||||
|
||||
Before a debugger touches anything, you need a map of what's running and a ledger of what you'll touch. Skipping either phase is how debug sessions turn into "why is my repo dirty a week later" sessions.
|
||||
|
||||
---
|
||||
|
||||
## Phase 0 — Environment Assessment
|
||||
|
||||
Map the ground truth before you attach. Attaching the wrong way wastes the first hour.
|
||||
|
||||
### 1. Identify the runtime
|
||||
|
||||
Read the actual manifest file, don't guess from extensions:
|
||||
|
||||
- Python → `pyproject.toml`, `requirements*.txt`, `setup.py`, `uv.lock`, `.python-version`
|
||||
- Node → `package.json` (check `scripts`, check `engines`, check `type: module`)
|
||||
- Rust → `Cargo.toml`, `rust-toolchain*`
|
||||
- Go → `go.mod`, `go.sum`
|
||||
- Native / mixed → `Makefile`, `CMakeLists.txt`, the binary itself (`file <path>`)
|
||||
|
||||
### 2. Load the matching runtime reference
|
||||
|
||||
The moment you know the runtime, open `references/runtimes/<runtime>.md`. The commands in this phase (and every phase after) are runtime-specific. The shape of the answers is the same; the commands are not.
|
||||
|
||||
### 3. Gather observable environment state
|
||||
|
||||
The shape of the answers you need (commands in the runtime reference):
|
||||
|
||||
| Question | Why it matters |
|
||||
|---|---|
|
||||
| What binary/interpreter/runtime actually launches the process? | Determines debugger flag plumbing. Wrappers (`tsx`, `poetry run`, `cargo run`, `bun`, supervisor scripts) change how flags propagate. |
|
||||
| Is there already a debug-relevant port in use, or another instance of the service running? | Either attach to it or kill it deliberately — never silently compete. |
|
||||
| Are symbols / source maps / debug info present and correct? | This determines whether breakpoints land on the right lines. Compiled-but-not-debug builds, stripped binaries, and incomplete source maps all silently misplace breakpoints. |
|
||||
| Does the code path require env vars, config files, or auth tokens to reach the bug? | Missing env often produces early-return paths that masquerade as the bug itself. |
|
||||
| Is there an existing failing test or known repro? | Prefer amplifying an existing repro over inventing one. |
|
||||
| Are watchers (file watchers, hot reloaders, supervisors) going to restart the process mid-session? | If yes, turn them off before attaching. Restarts drop inspector connections and invalidate breakpoints. |
|
||||
|
||||
### 4. Gate check
|
||||
|
||||
If any answer is "I'm not sure", you are not ready for Phase 1. Investigate until certain. Guessing here cascades into false-positive hypotheses in Phase 2.
|
||||
|
||||
---
|
||||
|
||||
## Phase 1 — Journal Setup
|
||||
|
||||
Open **one** journal file at the project root: `.debug-journal.md`. Single source of truth for every artifact this skill creates. The contract with the user that you can undo everything.
|
||||
|
||||
### Exclude from git (don't pollute the committed ignore list)
|
||||
|
||||
```bash
|
||||
grep -qx '.debug-journal.md' .git/info/exclude || echo '.debug-journal.md' >> .git/info/exclude
|
||||
```
|
||||
|
||||
`.git/info/exclude` is per-clone and not committed — perfect for local-session artifacts.
|
||||
|
||||
### Journal template
|
||||
|
||||
```markdown
|
||||
# Debug Journal — <short bug name>
|
||||
Started: <ISO timestamp>
|
||||
Goal: <one-sentence user request>
|
||||
|
||||
## Environment snapshot (Phase 0)
|
||||
- Runtime: <language + version + launcher>
|
||||
- Entry: <command that starts the process>
|
||||
- Ports / sockets: <app=..., debugger=..., etc>
|
||||
- Git HEAD: <sha>, working tree clean? <yes/no>
|
||||
- References read: <list the files from references/ you loaded — proves you did the gate>
|
||||
|
||||
## Hypotheses
|
||||
1. [STATUS] <hypothesis> — distinguishing evidence: <what would confirm/refute> — if true, fix is: <two words>
|
||||
2. ...
|
||||
|
||||
## Failed hypothesis round counter
|
||||
- Round 1: <result>
|
||||
- Round 2: <result>
|
||||
<!-- At 2 consecutive failures, invoke Oracle Triple (see 04-oracle-triple.md). -->
|
||||
|
||||
## Artifacts to revert
|
||||
<!-- Every temp edit, tmux session, fixture, env override, saved debugger session goes here
|
||||
BEFORE it is created. The rule is journal-then-modify. -->
|
||||
- [ ] `src/foo.py` — added `breakpoint()` on 2 lines. Revert: `git checkout src/foo.py`
|
||||
- [ ] tmux session `debug-server`. Kill: `tmux kill-session -t debug-server`
|
||||
- [ ] `/tmp/debug-payload.json`. Remove: `rm /tmp/debug-payload.json`
|
||||
- [ ] env var in current shell: `FOO_BASE_URL=...`. Unset when done.
|
||||
- [ ] GDB session save: `~/ghidra-projects/scratch.gzf`. Remove if not promoting.
|
||||
|
||||
## Findings
|
||||
<!-- Append observed values here with timestamp. Verbatim only, no paraphrasing. -->
|
||||
|
||||
## Oracle Triple (if invoked)
|
||||
<!-- One subsection per Oracle round, with the synthesized new hypothesis set. -->
|
||||
|
||||
## Final fix
|
||||
<!-- File paths + test path. Filled during Phase 7. -->
|
||||
```
|
||||
|
||||
### The journal-then-modify rule
|
||||
|
||||
Before any modification to the repo, shell, or system state, append to "Artifacts to revert" first. This one discipline is what prevents debug sessions from becoming git cleanup sessions.
|
||||
|
||||
If you catch yourself about to run a command that creates a file, opens a port, or modifies source — stop, journal the intended artifact with its revert command, then run the command. Not the other way around.
|
||||
|
||||
### Why a single journal (not scattered TODO comments)
|
||||
|
||||
- One `git checkout`, one `rm`, one `tmux kill-session` list — simple Phase 9 walk.
|
||||
- Survives interruptions. If you get pulled away mid-session, the next agent (or you later) can continue or revert without guessing.
|
||||
- Prevents the most common failure: leaving `console.log`/`print()`/`dbg!` scattered across the tree.
|
||||
@@ -0,0 +1,130 @@
|
||||
# Phase 2 + 3 — Hypothesis Formation & Parallel Investigation
|
||||
|
||||
One hypothesis is a hunch. Three hypotheses is a decision. Investigation is how you turn the decision into runtime evidence.
|
||||
|
||||
---
|
||||
|
||||
## Phase 2 — Hypothesis Formation (Minimum Three)
|
||||
|
||||
### Why three, not one
|
||||
|
||||
A single hypothesis creates confirmation bias: you'll read runtime state looking for evidence that confirms it and unconsciously discount contradictions. Three hypotheses force you to design queries that *distinguish* between them, which is the only way runtime evidence becomes decisive.
|
||||
|
||||
### Generate across orthogonal axes
|
||||
|
||||
If your three hypotheses are all variations of "the handler has a bug", you don't actually have three hypotheses. Span the space:
|
||||
|
||||
| Axis | Example framing |
|
||||
|---|---|
|
||||
| **User-code logic** | "The handler early-returns because condition X is unexpectedly true" |
|
||||
| **Library/SDK behavior** | "The third-party client swallows the error and returns a stub" |
|
||||
| **Environment/config** | "The env var is read at module-load time before it gets populated, so it's empty" |
|
||||
| **Async/timing** | "The promise rejects (or goroutine panics) after the response is already sent" |
|
||||
| **Silent side-effect** | "An earlier turn mutated shared state that the current turn inherits" |
|
||||
| **Observability gap** | "The error is raised but suppressed before logging; it only exists as an unawaited rejection / ignored signal" |
|
||||
| **Binary-level** (when applicable) | "The function we think is running is actually jumped over by a patched thunk / a different version loaded" |
|
||||
| **Build-vs-runtime** | "The code we're reading is not the code that's running — stale build, wrong symlink, cached wheel, or dist/ ahead of src/" |
|
||||
|
||||
### For each hypothesis, write in the journal
|
||||
|
||||
1. **Claim** — one sentence.
|
||||
2. **Distinguishing evidence** — the exact value or state that confirms or refutes it, AND where to read it (file:line, log source, breakpoint location, memory address).
|
||||
3. **If true, the fix is** — two words. Forces you to think through fix cost before committing to the hunt.
|
||||
|
||||
### Collapse rule
|
||||
|
||||
If two hypotheses have identical distinguishing evidence, they aren't actually different — collapse them and find a real alternative. If you can't come up with a third distinct hypothesis, you don't understand the system well enough yet. Go read a little more code before investigating.
|
||||
|
||||
---
|
||||
|
||||
## Phase 3 — Parallel Investigation
|
||||
|
||||
Branch depending on what's available.
|
||||
|
||||
### Path A: Team mode ENABLED
|
||||
|
||||
When the `team_*` tools are present, create a **debug-squad** team and split investigation across members working on different evidence sources. This is the right default whenever you have ≥3 hypotheses and any of them would take >10 minutes to investigate single-threaded.
|
||||
|
||||
**Team spec** — write to `~/.omo/teams/debug-squad/config.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "debug-squad",
|
||||
"lead": { "kind": "subagent_type", "subagent_type": "sisyphus" },
|
||||
"members": [
|
||||
{
|
||||
"kind": "category",
|
||||
"category": "deep",
|
||||
"prompt": "You are the Runtime State Inspector. Your job: attach to the live process, hit breakpoints, read program state (variables, heap, goroutines, stack, registers depending on runtime), and report observed values verbatim. Never guess — if you don't see the value, say so. Report back via team_send_message with file:line / address references and captured values. Never edit source code. Never run git commands. If you need an instrumentation statement added (breakpoint(), debugger;, dbg!, etc.), ask the Lead first."
|
||||
},
|
||||
{
|
||||
"kind": "category",
|
||||
"category": "deep",
|
||||
"prompt": "You are the Log Archaeologist. Your job: grep server logs, stderr streams, SDK-internal debug output (DEBUG env, RUST_LOG, GODEBUG, PYTHONASYNCIODEBUG), and correlate timestamps. Produce a timeline of events with latencies. Flag anything that looks like a silent catch, a swallowed rejection, a panic recovered-and-ignored, a success response that contains failure signals (HTTP 200 with empty body, stopReason=error, exit 0 with error-in-stdout). Never edit source code."
|
||||
},
|
||||
{
|
||||
"kind": "category",
|
||||
"category": "deep",
|
||||
"prompt": "You are the Reproduction Engineer. Your job: build the smallest reliable repro — a curl command, a vitest/pytest/go test, a tmux script, a Playwright script for browser bugs, a pwntools script for binary targets. It must reproduce on first try and be copy-pasteable by the Lead. Document exact input, expected output, observed output. Save repro artifacts under /tmp/ and tell the Lead to journal them. If the bug is browser-based you MUST use Playwright CLI — do not simulate with curl."
|
||||
},
|
||||
{
|
||||
"kind": "category",
|
||||
"category": "deep",
|
||||
"prompt": "You are the Trace Correlator. Your job: take findings from the other members and cross-link them. Build a causal chain from symptom to suspected cause. Identify missing evidence. Propose the next single most-decisive runtime query. Never edit source code; only reason across already-captured evidence. If hypotheses diverge sharply after correlation, tell the Lead immediately — that is the signal for the Oracle Triple."
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Assignment rule**: one hypothesis → one `team_task_create`. Give each hypothesis to the member whose evidence source is most likely to confirm or refute it. Broadcast the full hypothesis list once via `team_send_message(to="*")` so members know what the others are testing.
|
||||
|
||||
**Lead responsibilities**:
|
||||
- Maintain the journal (members do not write to it).
|
||||
- Approve any source-code edits (including `debugger;` / `breakpoint()` / `dbg!` statements).
|
||||
- Synthesize member reports into updated hypothesis statuses.
|
||||
- Decide when to disband: `team_shutdown_request` → `team_approve_shutdown` → `team_delete`.
|
||||
|
||||
**Team does NOT include Oracle** — Oracle is a hard-reject team member type. Oracle is used separately in Phase 4 (see `04-oracle-triple.md`).
|
||||
|
||||
### Path B: Team mode DISABLED
|
||||
|
||||
Fan out async explore/deep subagents instead. Same rule: one hypothesis per subagent.
|
||||
|
||||
```
|
||||
task(subagent_type="explore", load_skills=[], run_in_background=true,
|
||||
prompt="[CONTEXT: bug summary + which hypothesis you own + what state to look at]
|
||||
Runtime state investigation for hypothesis 1: ...")
|
||||
task(subagent_type="explore", load_skills=[], run_in_background=true,
|
||||
prompt="Log/timing investigation for hypothesis 2: ...")
|
||||
task(category="deep", load_skills=[], run_in_background=true,
|
||||
prompt="Reproduction minimizer for hypothesis 3: ...")
|
||||
```
|
||||
|
||||
End your response, wait for completion notifications, then synthesize.
|
||||
|
||||
---
|
||||
|
||||
## Evidence capture discipline (both paths)
|
||||
|
||||
For every piece of runtime state captured, record in the journal:
|
||||
|
||||
```markdown
|
||||
### <ISO timestamp> — <what you looked at>
|
||||
- Source: <file:line | log source | curl command | breakpoint address>
|
||||
- Value: `<verbatim>`
|
||||
- Interpretation: <one line — why this matters>
|
||||
- Refutes/Confirms: H<n>
|
||||
```
|
||||
|
||||
**Verbatim values only. No paraphrasing.**
|
||||
|
||||
- `messages.length=0` is evidence.
|
||||
- "messages seemed empty" is not evidence — it's a memory of an observation, and memory of observations is where debug sessions go to die.
|
||||
|
||||
If you find yourself about to paraphrase, stop, go back, and copy the raw value.
|
||||
|
||||
---
|
||||
|
||||
## Round completion
|
||||
|
||||
A "round" is complete when every hypothesis has either confirming or refuting evidence — or when you have exhausted the evidence sources available without a decisive result. If the round ends inconclusively, that counts as a failed round for the counter in the journal. See `04-oracle-triple.md` for what to do at 2 consecutive failed rounds.
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
# Phase 4 — Oracle Triple Consultation
|
||||
|
||||
At 2 consecutive failed hypothesis rounds, stop investigating and reframe. Continuing past two failures usually means the real cause is in a category you haven't imagined — and more time on your current mental model is wasted time.
|
||||
|
||||
The Oracle Triple is how you break out of the mental box.
|
||||
|
||||
> ⚠️ **Wrong tool for non-debugging tasks.** The Triple is for *stuck root-cause hunts*. If your task is producing an artifact (extraction, reverse engineering, audit, compliance documentation) and you want a skeptical review before declaring it done, use the **Verification Oracle** pattern in [partial-runtime-evidence.md](partial-runtime-evidence.md#verification-oracle-pattern-for-non-debug-tasks). Running the Triple on a finished extraction returns three diverging "what if you tried…" tangents that are not what you need.
|
||||
|
||||
---
|
||||
|
||||
## When to invoke
|
||||
|
||||
| Situation | Invoke? |
|
||||
|---|---|
|
||||
| 1 round failed, you have new distinguishing evidence | No — run one more round with a refined hypothesis set |
|
||||
| 2 rounds failed, hypotheses now feel like variations of each other | **Yes — invoke now** |
|
||||
| 2 rounds failed, no new evidence angles left to try | **Yes — invoke now** |
|
||||
| You've been investigating >2 hours on the same bug | **Yes — invoke now regardless of round count** |
|
||||
| 1 round failed but the user is watching and wants speed | No — one round isn't enough to justify Oracle cost. Resist the urge. |
|
||||
|
||||
---
|
||||
|
||||
## Why three Oracles, and why *orthogonal* framings
|
||||
|
||||
A single Oracle call returns a single coherent analysis. Coherent analyses tend to inherit the framing of the prompt, which means they inherit the same blind spots the investigator already has. Three Oracles with *orthogonal framings* force the analyses to diverge, and the places where they agree across frames is where the real signal lives.
|
||||
|
||||
The three framings below are chosen to cover distinct bug-cause categories:
|
||||
|
||||
- **A (obvious-but-missed)** — embarrassingly simple causes the investigator walked past.
|
||||
- **B (system-boundary)** — causes living at integration seams, not in the code being read.
|
||||
- **C (invariant-violation)** — assumptions load-bearing to current hypotheses that may themselves be false.
|
||||
|
||||
Spawn all three in parallel.
|
||||
|
||||
---
|
||||
|
||||
## The three prompts
|
||||
|
||||
```
|
||||
task(subagent_type="oracle", load_skills=[], run_in_background=true,
|
||||
prompt="[CONTEXT: bug description + evidence captured so far, verbatim, with file:line refs]
|
||||
|
||||
Framing A — OBVIOUS-BUT-MISSED.
|
||||
What is the most embarrassing, most obvious cause that a senior engineer would spot in 30 seconds and we've overlooked? Consider:
|
||||
- typos, off-by-one
|
||||
- wrong variable name / wrong constant / wrong import
|
||||
- stale cache, wrong file edited, wrong process inspected
|
||||
- attached to the wrong instance of the service
|
||||
- test harness running different code than the app
|
||||
- editing src/ while running dist/
|
||||
|
||||
Give me exactly three candidate causes ranked by likelihood, with one sentence each explaining why our evidence is consistent with each.")
|
||||
|
||||
task(subagent_type="oracle", load_skills=[], run_in_background=true,
|
||||
prompt="[CONTEXT: bug description + evidence captured so far]
|
||||
|
||||
Framing B — SYSTEM-BOUNDARY.
|
||||
What if the bug is NOT in the code we've been reading, but at a boundary? Consider:
|
||||
- third-party SDK behavior that contradicts its docs
|
||||
- middleware that mutates the request or response
|
||||
- a proxy/gateway/load balancer that rewrites headers or bodies
|
||||
- build-time vs runtime env-var resolution
|
||||
- module-load-order issue
|
||||
- shared-library version mismatch (system lib vs bundled lib)
|
||||
- ABI difference (native addons, glibc versions, musl vs glibc)
|
||||
- wrong transport (HTTP/1.1 vs HTTP/2, TLS version negotiation)
|
||||
|
||||
Give me three candidate causes, each naming the specific boundary and the specific contract assumption that might be violated.")
|
||||
|
||||
task(subagent_type="oracle", load_skills=[], run_in_background=true,
|
||||
prompt="[CONTEXT: bug description + evidence captured so far]
|
||||
|
||||
Framing C — INVARIANT-VIOLATION.
|
||||
Which invariants that we've been ASSUMING TRUE might actually be false?
|
||||
Enumerate the five assumptions most load-bearing to our current hypotheses, then for each:
|
||||
- describe the smallest runtime query that would falsify it
|
||||
- predict what the observable would be if the invariant holds vs if it fails
|
||||
|
||||
We want at least one of these queries to be decisive.")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Synthesizing across three Oracles
|
||||
|
||||
**Do not pick the highest-ranked candidate from a single Oracle.** That defeats the purpose of getting three framings.
|
||||
|
||||
Instead, walk the outputs in this order:
|
||||
|
||||
### 1. Agreement scan
|
||||
|
||||
Note which candidate causes appear in at least two Oracles' outputs. Independent agreement across orthogonal framings is strong signal — when the obvious-but-missed framing and the system-boundary framing both land on the same cause, that's usually the bug.
|
||||
|
||||
### 2. Disagreement scan
|
||||
|
||||
Note where Oracles disagree. Disagreement is genuine uncertainty that runtime evidence (not more reasoning) must resolve. Each disagreement becomes a candidate for the next round's distinguishing query.
|
||||
|
||||
### 3. New falsification queries
|
||||
|
||||
Framing C produces concrete "one query that would decide it" suggestions. Pull these verbatim into your new round's evidence-gathering plan — they are designed to be decisive.
|
||||
|
||||
### 4. Build the new hypothesis set
|
||||
|
||||
Minimum 3, same rules as Phase 2. Aim to have hypotheses drawn from the agreement scan (likely cause) AND from the disagreement scan (so one round's evidence resolves the disagreement).
|
||||
|
||||
Record in the journal:
|
||||
|
||||
```markdown
|
||||
## Oracle Triple — Round <N>
|
||||
- Invoked at: <ISO timestamp>
|
||||
- Framing A summary: <top 3 candidates, one line each>
|
||||
- Framing B summary: <top 3 candidates>
|
||||
- Framing C summary: <5 load-bearing assumptions + falsification queries>
|
||||
|
||||
### Cross-framing agreement
|
||||
- <candidate> appeared in A + B
|
||||
- <candidate> appeared in B + C
|
||||
|
||||
### New hypothesis set
|
||||
1. <hypothesis> — evidence to gather: <one-liner>
|
||||
2. ...
|
||||
```
|
||||
|
||||
### 5. Reset the counter
|
||||
|
||||
Reset the "consecutive failed rounds" counter to 0. Return to Phase 3 (parallel investigation) with the new set.
|
||||
|
||||
---
|
||||
|
||||
## If *another* 2 rounds fail after the Oracle Triple
|
||||
|
||||
You are genuinely stuck. This is the escalation threshold.
|
||||
|
||||
Escalate to the user (see `05-escalate.md`) with the full trace: every hypothesis tried, every piece of evidence captured, both Oracle syntheses. Do not guess a fix.
|
||||
|
||||
This is rare — in practice, the Oracle Triple resolves almost all stuck debugging sessions within one round, because it pulls in framings the investigator was too close to the code to see.
|
||||
@@ -0,0 +1,69 @@
|
||||
# Phase 5 — User Decision Escalation
|
||||
|
||||
Escalation is for genuine ambiguity, not for skipping investigation. Most "should I ask the user" moments are really "I don't want to do one more query" moments, and those are wrong.
|
||||
|
||||
---
|
||||
|
||||
## Ask the user ONLY when
|
||||
|
||||
- **Evidence exhausted**, contradictions remain, and further investigation would require a decision with policy implications (e.g. "patch the third-party SDK vs wrap it vs change architecture").
|
||||
- The bug has **multiple valid fixes with different scope/risk tradeoffs** and the user's preference drives the choice.
|
||||
- A proposed fix would **change observable product behavior** for the end user (not just fix the internal bug).
|
||||
- You've **exhausted the Oracle Triple** and another 2 rounds failed after synthesis.
|
||||
|
||||
## Do NOT ask when
|
||||
|
||||
- You haven't tried the Oracle Triple yet.
|
||||
- The question can be answered by one more runtime query.
|
||||
- You're asking for permission to do the obvious thing.
|
||||
- You're asking because you're tired.
|
||||
|
||||
---
|
||||
|
||||
## Escalation format (paste into the reply)
|
||||
|
||||
Keep it short. Evidence-dense. One decision, not a status update.
|
||||
|
||||
```markdown
|
||||
## Decision needed
|
||||
|
||||
**What we know** (verbatim evidence, not paraphrase):
|
||||
- <fact 1 with file:line or address>
|
||||
- <fact 2 with source>
|
||||
- <what the evidence rules IN>
|
||||
- <what the evidence rules OUT>
|
||||
|
||||
**What the decision is** (one sentence):
|
||||
<the fork in the road>
|
||||
|
||||
**Options**:
|
||||
|
||||
| # | Fix | Scope | Risk | Effort |
|
||||
|---|-----|-------|------|--------|
|
||||
| A | <short label> | <files touched / layers> | <regressions possible> | <rough> |
|
||||
| B | ... | ... | ... | ... |
|
||||
| C | ... | ... | ... | ... |
|
||||
|
||||
**Recommendation**: <A/B/C> because <one-sentence reason>.
|
||||
|
||||
Which direction do you want?
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Anti-patterns in escalation
|
||||
|
||||
- **Asking without evidence.** "What do you want me to do?" is not an escalation, it's abandonment. Every escalation includes the evidence the user needs to decide.
|
||||
- **Two questions in one.** One decision per escalation. Multi-part questions lead to partial answers and re-escalation.
|
||||
- **Escalating before Phase 4.** If you haven't tried the Oracle Triple, you haven't earned the right to escalate.
|
||||
- **Presenting options you don't actually have.** If option C requires a library the user doesn't use, don't list it. The options are only things you can actually do today.
|
||||
- **Hiding a recommendation.** The user hired you to think — always end with a recommendation, even if you're low-confidence. Say so explicitly: "Recommendation (low confidence): B, because X. If you have context about Y that I don't, it might change to A."
|
||||
|
||||
---
|
||||
|
||||
## What happens after the user responds
|
||||
|
||||
- **User picks an option**: return to Phase 6 (root cause confirmation) with the chosen direction. The user's choice is not itself confirmation — you still need runtime evidence that the cause you're fixing is the cause in play.
|
||||
- **User proposes a different option you hadn't considered**: treat it as new information. Update hypotheses. May trigger another Phase 3 round.
|
||||
- **User gives more context that resolves the disagreement**: skip to Phase 6.
|
||||
- **User is also unsure**: that's a signal you need more evidence, not more opinions. Run one more targeted query before asking again.
|
||||
@@ -0,0 +1,116 @@
|
||||
# Phase 6 + 7 — Root Cause Confirmation & TDD Fix
|
||||
|
||||
A cause is not "confirmed" until you can toggle the bug by toggling the cause. Every other level of evidence is correlation, and correlation-driven fixes ship bugs.
|
||||
|
||||
---
|
||||
|
||||
## Phase 6 — Root Cause Confirmation
|
||||
|
||||
You are allowed to call the cause "confirmed" only when ALL THREE of these hold:
|
||||
|
||||
### 1. Captured runtime value matches the hypothesis exactly
|
||||
|
||||
Not "the value looks consistent with" — the value is exactly the value the hypothesis predicted. If your hypothesis was "baseUrl is api.anthropic.com despite ANTHROPIC_BASE_URL being set to a proxy", the captured value is literally `"https://api.anthropic.com"` in the debugger at the moment of the HTTP call.
|
||||
|
||||
### 2. Reproducible
|
||||
|
||||
Running the repro a second time yields the same observation. Flaky repros mean you haven't isolated the cause; you've isolated a symptom that sometimes appears when the cause does. Keep investigating.
|
||||
|
||||
### 3. Toggle proof (the one most skipped)
|
||||
|
||||
**Changing the value** (via debugger assignment, env override, or a speculative one-line patch) **makes the bug disappear — and reverting brings the bug back**.
|
||||
|
||||
If you can't toggle the bug by toggling the suspected cause, what you have is a correlation, not a mechanism. A correlation is a strong hypothesis, not a confirmed cause.
|
||||
|
||||
Examples of a valid toggle proof:
|
||||
|
||||
| Suspected cause | Toggle |
|
||||
|---|---|
|
||||
| Env var overrides library default, and the override is wrong | Unset the env var → bug goes away. Reset it → bug comes back. |
|
||||
| Async task is not awaited | Add `await` → bug goes away. Remove `await` → bug comes back. |
|
||||
| Third-party SDK uses hardcoded URL | Monkey-patch SDK to use env URL → bug goes away. Unpatch → bug comes back. |
|
||||
| Race condition on shared state | Add a mutex → bug goes away under load. Remove mutex → bug comes back under load. |
|
||||
|
||||
If you can't construct a toggle proof, you haven't confirmed the cause. Run one more round.
|
||||
|
||||
### Update the journal
|
||||
|
||||
```markdown
|
||||
## Root cause (confirmed <ISO timestamp>)
|
||||
- Mechanism: <one paragraph, causal not correlational — the chain from cause to observable symptom>
|
||||
- Evidence: <file:line of captured value | path to saved repro | address + register state>
|
||||
- Toggle proof: "With <change X>, repro produces <good>. Reverting <change X>, repro produces <bad>."
|
||||
- Fix scope: <files and approximate line count>
|
||||
```
|
||||
|
||||
The "mechanism" field is the acid test. If you can't write the causal chain from cause to observable symptom as one paragraph, you don't yet understand the bug well enough to fix it.
|
||||
|
||||
---
|
||||
|
||||
## Phase 7 — TDD Fix
|
||||
|
||||
Red, green, refactor. No shortcuts.
|
||||
|
||||
### 1. Red — failing-first test
|
||||
|
||||
Write a test that fails *specifically because of this bug*. Requirements:
|
||||
|
||||
- **Test name reads like a bug report.** `test_refinement_turn_returns_empty_content_when_anthropic_returns_401` is good. `test_bug_fix` is not.
|
||||
- **Failure message clearly shows what the bug looks like.** If someone reads only the failure output, they understand what's broken.
|
||||
- **Minimum infrastructure.** Don't spin up the whole server if a unit test against the right seam captures the mechanism.
|
||||
|
||||
Run the test. Confirm it fails. Paste the failure output into the journal:
|
||||
|
||||
```markdown
|
||||
### Red phase (<ISO timestamp>)
|
||||
Test: <path>::<name>
|
||||
Command: <exact invocation>
|
||||
Output:
|
||||
```
|
||||
<verbatim failure output>
|
||||
```
|
||||
Confirms: the bug is reproducible at the test-harness level, not just the manual repro.
|
||||
```
|
||||
|
||||
### 2. Green — minimum change
|
||||
|
||||
Make the test pass with the **smallest change that fully fixes the observed mechanism**.
|
||||
|
||||
If the diff is larger than ~30 lines and you aren't refactoring, something is wrong — either you're fixing more than the bug, or the root cause was deeper than you confirmed. Back to Phase 6.
|
||||
|
||||
Signs you're over-fixing:
|
||||
- Adding "just in case" null checks or try/except around other code
|
||||
- Refactoring adjacent functions because "while I'm here"
|
||||
- Adding new configuration options the bug didn't require
|
||||
- Introducing new abstractions to "make this cleaner"
|
||||
|
||||
Resist all of these. Fix the bug. Note the surrounding issues for follow-up. Move on.
|
||||
|
||||
### 3. Refactor — ONLY AFTER GREEN
|
||||
|
||||
Only cleanup directly related to the fix. Do not re-architect.
|
||||
|
||||
If the code around the fix is rough, note it in the journal as a follow-up for the user; do not expand scope here. Refactoring during a bugfix is how one-line fixes turn into hundred-line diffs nobody can review.
|
||||
|
||||
### 4. Regression — full suite green
|
||||
|
||||
Run the full test suite for the affected package (not just the one new test). Existing tests must still pass.
|
||||
|
||||
If they don't, your "fix" broke something else. Back to Phase 6 with the new failure as evidence — usually it means the mechanism you thought you fixed was load-bearing for some other code path you didn't know about, and the "broken" test is actually pointing at a better understanding of the system.
|
||||
|
||||
### Update the journal
|
||||
|
||||
```markdown
|
||||
### Green phase (<ISO timestamp>)
|
||||
Fix: <file:line> — <two-line description of the change>
|
||||
Test: <path>::<name> now passes
|
||||
Full suite: <N tests, <M failures — should be 0>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## The red-green discipline summary
|
||||
|
||||
No red test → no proof the fix addresses the reported bug. Only proof it doesn't break tests that already existed.
|
||||
|
||||
A test written *after* the fix might still pass with the fix reverted. If that's the case, the test doesn't lock the bug — it locks something else. Always verify the test fails without the fix and passes with it. The journal should show both outputs.
|
||||
@@ -0,0 +1,94 @@
|
||||
# Phase 8 — Manual QA by Actually Using It
|
||||
|
||||
Tests cover cases you thought of. Real usage covers the ones you didn't.
|
||||
|
||||
The single fastest way to ship a broken fix is to stop at "tests pass". Manual QA means interacting with the running system the way the user does, then comparing observed behavior to the original bug report.
|
||||
|
||||
---
|
||||
|
||||
## Product-type playbook
|
||||
|
||||
Pick the row that matches the product. Do what it says. Do not substitute.
|
||||
|
||||
| Product type | QA means… |
|
||||
|---|---|
|
||||
| **CLI tool** | Open `tmux`, run the actual command end-to-end, capture output. Paste the session transcript into the journal. Include exit code, stdout, stderr, side-effect check (files created/modified). |
|
||||
| **HTTP API** | Start the real server, hit endpoints with `curl` or `httpie`, inspect response status + body + headers. Hit the specific endpoint that reproduced the bug. If there's auth, use real auth. |
|
||||
| **Browser-served web app** | **Drive a real browser via Playwright CLI.** See [tools/playwright-cli.md](../tools/playwright-cli.md). Navigate the exact page/flow that reproduced the bug. Capture screenshot + DOM + network evidence. **Do not substitute with curl** — browsers have state (cookies, localStorage, service workers, client-side JS, viewport-dependent CSS) that curl does not have. |
|
||||
| **Agent / LLM pipeline** | Run the same user prompt that originally failed. Capture the full turn — tool calls, messages, usage counters. **Confirm non-zero usage** (zero usage = still failing silently, see silent-failure check below). |
|
||||
| **Background worker / job queue** | Trigger the job through the normal entry point (API call, cron tick, message publish), tail the worker logs, observe completion state in the queue or DB. Don't just call the worker function directly — the trigger path matters. |
|
||||
| **MCP server** | Invoke the tool via its actual client (Claude Desktop, Cursor, etc. if available) or `mcp-cli`, not just the HTTP probe endpoint. The MCP handshake itself is sometimes where bugs live. |
|
||||
| **Native binary** | Re-run the exact command that crashed / misbehaved. If the input was a file, use the same file. If the bug was exploitable, confirm the exploit repro via pwntools (see [tools/pwntools.md](../tools/pwntools.md)). Capture exit code, signal if any, core dump if generated. |
|
||||
| **Bundled-app binary** (Bun SEA, Node SEA, Electron, etc.) | Re-run the exact command. If the operation requires paid quota / blocked network, capture the **app's debug log** (`APP_DEBUG=1 APP_LOG_LEVEL=debug APP_LOG_FILE=/tmp/trace.log`) which usually emits the assembled request before sending. See [methodology/partial-runtime-evidence.md](partial-runtime-evidence.md) for combining partial signals into a defensible verification. |
|
||||
| **Long-running daemon** | Start fresh, let it run for the amount of time the bug originally took to manifest (not less), capture resource usage (memory, fd, cpu) throughout. Short-running QA misses resource leaks and cumulative state bugs. |
|
||||
|
||||
---
|
||||
|
||||
## Journal format
|
||||
|
||||
Every QA run goes in the journal under "Findings":
|
||||
|
||||
```markdown
|
||||
### Manual QA — <product type> (<ISO timestamp>)
|
||||
- Scenario: <one line describing what you did>
|
||||
- Command: `<exact invocation>`
|
||||
- Observed output:
|
||||
```
|
||||
<verbatim output, trimmed to relevant section>
|
||||
```
|
||||
- Expected output: <what correct behavior looks like>
|
||||
- Fix verified: yes / no / partial — <details>
|
||||
```
|
||||
|
||||
If any QA step shows **partial or regressed behavior**, this is not "mostly done" — it's incomplete. Return to Phase 6.
|
||||
|
||||
---
|
||||
|
||||
## The silent-failure check (always run)
|
||||
|
||||
Regardless of product type, audit the fix against these silent-failure patterns. If the original bug was a silent failure, the same pattern may exist in adjacent code that you haven't tested yet.
|
||||
|
||||
### Universal silent-failure signals
|
||||
|
||||
- HTTP 2xx with empty or default body
|
||||
- Response `ok: true` but a sub-field contains an error token (e.g. `stopReason: "error"`, `status: "failed"`)
|
||||
- `usage.totalTokens === 0` on an LLM response
|
||||
- Process exit code 0 but stderr contains an exception traceback
|
||||
- Panic recovered and logged but ignored
|
||||
- Goroutine / task / promise rejection with no top-level handler
|
||||
- `try { ... } catch { /* swallowed */ }` or `except: pass`
|
||||
- Success response shape but semantic field indicates failure (e.g. `error: null` actually being `error: "..."` with falsy check)
|
||||
- Write returned success but read-back shows stale data
|
||||
- Job marked complete but side-effect did not happen
|
||||
- Cache hit path returned stale data and no refresh was triggered
|
||||
|
||||
### Language-specific silent-failure signals
|
||||
|
||||
Check the runtime reference for additional patterns:
|
||||
|
||||
- [runtimes/python.md](../runtimes/python.md) — asyncio task exceptions, bare `except`, `logging.exception` that goes nowhere
|
||||
- [runtimes/node.md](../runtimes/node.md) — unhandled promise rejections, `void` on async, swallowed `.catch(() => {})`
|
||||
- [runtimes/rust.md](../runtimes/rust.md) — `.unwrap_or_default()`, `let _ = result`, error variants discarded
|
||||
- [runtimes/go.md](../runtimes/go.md) — `if err != nil { return err }` that never reaches user output, recovered panics, buffered channels that block silently
|
||||
- [runtimes/native-binary.md](../runtimes/native-binary.md) — ignored return codes from libc, missing `perror`, `alarm()` / signal masks
|
||||
- [runtimes/bundled-js-binary.md](../runtimes/bundled-js-binary.md) — `process.env.X` baked at build time, dead code from tree-shaking failures, worker sub-bundles diverging from main bundle
|
||||
|
||||
### What to do when you find another silent-failure spot
|
||||
|
||||
Don't fix it. This is out of scope for the current bug.
|
||||
|
||||
Note it in the journal under a "Follow-ups" section with:
|
||||
- File:line
|
||||
- Pattern matched
|
||||
- Proposed fix sketch (one line)
|
||||
- Risk level (what happens if left unfixed)
|
||||
|
||||
Surface these to the user in the final message under "Next steps I didn't take".
|
||||
|
||||
---
|
||||
|
||||
## The "fix verified" bar
|
||||
|
||||
"Fix verified" means: the exact original failing scenario, re-run, now produces the correct output. Not a similar scenario. Not a unit test of the fix. The original scenario.
|
||||
|
||||
If you can't re-run the original scenario (e.g. it required a specific data state that's gone), construct the closest equivalent and document the difference in the journal. Escalate to the user if the equivalent is materially different.
|
||||
@@ -0,0 +1,164 @@
|
||||
# Phase 9 + 10 — Cleanup & Final Verification
|
||||
|
||||
The working tree after the session must differ from before only by the real fix and its test. Anything else is a process failure.
|
||||
|
||||
---
|
||||
|
||||
## Phase 9 — Cleanup & Revert
|
||||
|
||||
### The walk
|
||||
|
||||
Open the journal's "Artifacts to revert" list. Walk it top to bottom. Check each box only after the revert command succeeds and produces no error.
|
||||
|
||||
### Standard revert operations
|
||||
|
||||
Most sessions create some combination of these artifacts. The commands below are the defaults — your journal should have the exact commands for this session.
|
||||
|
||||
```bash
|
||||
# --- Temporary source edits (instrumentation statements, debug prints) ---
|
||||
git checkout <file> # reverts only that file
|
||||
git diff <file> # verify clean
|
||||
|
||||
# --- tmux sessions ---
|
||||
tmux kill-session -t <session-name>
|
||||
tmux ls # confirm gone
|
||||
|
||||
# --- Temp fixtures / scratch scripts ---
|
||||
rm -f /tmp/debug-*.*
|
||||
ls /tmp/debug-*.* 2>/dev/null # confirm gone (ls returns non-zero when no match)
|
||||
|
||||
# --- Background processes (debugger-attached runtimes) ---
|
||||
pkill -f 'node --inspect' || true
|
||||
pkill -f 'python -m pdb' || true
|
||||
pkill -f 'debugpy' || true
|
||||
pkill -f 'dlv' || true
|
||||
pkill -f 'gdb' || true
|
||||
pkill -f 'lldb' || true
|
||||
|
||||
# --- Debug-relevant ports confirmed free ---
|
||||
lsof -iTCP:9229 -sTCP:LISTEN -nP 2>/dev/null # Node inspector default
|
||||
lsof -iTCP:5678 -sTCP:LISTEN -nP 2>/dev/null # debugpy default
|
||||
lsof -iTCP:2345 -sTCP:LISTEN -nP 2>/dev/null # dlv default
|
||||
lsof -iTCP:9999 -sTCP:LISTEN -nP 2>/dev/null # pwndbg/gdb-server default
|
||||
|
||||
# --- Env var overrides in current shell ---
|
||||
unset DEBUG_OVERRIDE_FOO
|
||||
unset PYTHONBREAKPOINT
|
||||
unset RUST_LOG
|
||||
unset DEBUG
|
||||
|
||||
# --- Ghidra scratch projects (if created just for this session) ---
|
||||
# rm -rf ~/ghidra-projects/debug-scratch
|
||||
|
||||
# --- Core dumps from debugging (if any) ---
|
||||
rm -f ./core ./core.* ~/core.*
|
||||
|
||||
# --- Playwright trace files ---
|
||||
rm -rf playwright-report/ test-results/
|
||||
```
|
||||
|
||||
### The verify command
|
||||
|
||||
This is the single most important check of the whole skill:
|
||||
|
||||
```bash
|
||||
git status
|
||||
git diff --stat
|
||||
```
|
||||
|
||||
The diff must contain **only**:
|
||||
|
||||
1. The real fix.
|
||||
2. The new failing-first test.
|
||||
3. Nothing else.
|
||||
|
||||
### Detector checklist — scan the diff for these
|
||||
|
||||
If `git status` shows any untracked debug file, or `git diff` shows any of the patterns below, **you are not done**. Clean it.
|
||||
|
||||
| Pattern | Usually means |
|
||||
|---|---|
|
||||
| `debugger;` | Node debug statement left behind |
|
||||
| `breakpoint()` | Python debug statement left behind |
|
||||
| `dbg!(...)` | Rust debug macro left behind |
|
||||
| `fmt.Println("DEBUG: ...")` | Go ad-hoc print |
|
||||
| `console.log("[DEBUG]` | Node ad-hoc log |
|
||||
| `print(f"DEBUG: ` | Python ad-hoc print |
|
||||
| `// TODO DEBUG`, `// HACK`, `// XXX` | Stale debug marker |
|
||||
| `// <PROJECT>-DEBUG` | Session-specific marker from this skill's edits |
|
||||
| Commented-out code blocks near the fix | Dead code from trial fixes |
|
||||
| Reordered imports or formatting in unrelated files | Drift from your editor's autoformat during the session |
|
||||
|
||||
### Remove the journal
|
||||
|
||||
Only once the git check is clean:
|
||||
|
||||
```bash
|
||||
rm .debug-journal.md
|
||||
sed -i.bak '/^\.debug-journal\.md$/d' .git/info/exclude && rm -f .git/info/exclude.bak
|
||||
```
|
||||
|
||||
The journal is not part of the fix; it doesn't belong in the commit or in the git exclude list.
|
||||
|
||||
---
|
||||
|
||||
## Phase 10 — Final Verification
|
||||
|
||||
Last gate before reporting done. All four gates must be true, and all four must have **evidence in your final message** to the user. Passing a gate without evidence is the same as failing it.
|
||||
|
||||
### The four gates
|
||||
|
||||
1. **Red→green toggle confirmed** — show the failing test output from before the fix and passing output after. Both outputs visible in the reply or the journal.
|
||||
|
||||
2. **Full test suite green** — show the suite's final pass line (e.g. `42 passed in 3.14s`). Not just the new test.
|
||||
|
||||
3. **Manual QA reproduced the fix** — show the command or scenario that originally failed and its now-correct output. Verbatim, not paraphrased.
|
||||
|
||||
4. **Working tree clean of debug artifacts** — show `git diff --stat` output containing only fix + test, plus `git status` clean of untracked debug files.
|
||||
|
||||
If any of the four lacks evidence, you have not finished — return to the appropriate phase.
|
||||
|
||||
### Final message template
|
||||
|
||||
Keep it short. Evidence-dense. The user should be able to skim it in 30 seconds.
|
||||
|
||||
```markdown
|
||||
Fixed.
|
||||
|
||||
**Root cause**: <one sentence — the mechanism, not the symptom>
|
||||
**Fix**: `<file:line>` — <two words>
|
||||
**Test**: `<test file>::<test name>` — red without fix, green with fix
|
||||
**QA**: <one line describing what you ran and what you saw>
|
||||
|
||||
Diff:
|
||||
```
|
||||
<git diff --stat output — should be tiny>
|
||||
```
|
||||
|
||||
**Next steps I didn't take** (awaiting your decision):
|
||||
- <follow-up 1, if any — from QA silent-failure scan or refactor opportunities noted during Phase 7>
|
||||
- <follow-up 2 — or "none" if nothing else surfaced>
|
||||
```
|
||||
|
||||
### Example (from a real session)
|
||||
|
||||
```markdown
|
||||
Fixed.
|
||||
|
||||
**Root cause**: pi-mono Agent's `model.baseUrl` was hardcoded to `api.anthropic.com`, so the `ANTHROPIC_BASE_URL` env var was silently ignored. The proxy API key was rejected by the real Anthropic API with 401, but pi-mono packaged the error into the assistant message's `errorMessage` field instead of throwing, so the route's try/catch never fired and the client received HTTP 200 with empty content.
|
||||
|
||||
**Fix**: `core/pi-bridge/modelResolver.ts:117` — override baseUrl
|
||||
**Test**: `__tests__/core/modelResolver.test.ts::resolves_env_override` — red without fix, green with fix
|
||||
**QA**: `curl -X POST /api/refinement/chat` with proxy env set, observed non-zero usage and non-empty content
|
||||
|
||||
Diff:
|
||||
```
|
||||
core/pi-bridge/modelResolver.ts | 3 +++
|
||||
__tests__/core/modelResolver.test.ts | 42 ++++++++++++++++++++++
|
||||
2 files changed, 45 insertions(+)
|
||||
```
|
||||
|
||||
**Next steps I didn't take** (awaiting your decision):
|
||||
- pi-mono itself silently swallows LLM errors into `errorMessage`; adding a throw-on-error wrapper at our orchestrator layer would surface these upstream
|
||||
- Same silent-failure pattern exists in the planning route — likely the same fix applies
|
||||
```
|
||||
+229
@@ -0,0 +1,229 @@
|
||||
# Partial Runtime Evidence — When You Cannot Execute the Real Operation
|
||||
|
||||
Read this when **runtime truth beats code reading** is in conflict with **you cannot run the actual operation**.
|
||||
|
||||
The skill's first invariant is "runtime state is the only source of truth." But sometimes the only state you can produce is a *partial* observation — the real call requires paid credits, a hardware device you don't have, network access through a corporate proxy, a production secret, or a customer dataset.
|
||||
|
||||
**Partial runtime evidence is still runtime evidence.** This reference tells you which partial signals to harvest and how to combine them so the conclusion is defensible.
|
||||
|
||||
---
|
||||
|
||||
## When this applies
|
||||
|
||||
Use this reference when ALL are true:
|
||||
|
||||
1. The bug or extraction question requires runtime confirmation (per skill invariant #1).
|
||||
2. You attempted the obvious "just run it" path and it failed for reasons unrelated to the bug:
|
||||
- 401/402/403 from a paid API
|
||||
- "device not found" / "permission denied" / SIP block
|
||||
- Production-only credentials
|
||||
- Network isolation (air-gapped, behind VPN you don't have)
|
||||
- Time-of-day or quota limits
|
||||
3. **Mocking the entire system** would defeat the verification — you specifically need evidence about how the *real* code behaves, not a stub.
|
||||
|
||||
If only #1 and #2 are true and you can mock cleanly, just mock and proceed. This file is for cases where mocking would invalidate the answer.
|
||||
|
||||
---
|
||||
|
||||
## The hierarchy of partial evidence (strongest first)
|
||||
|
||||
When you cannot capture the full outbound payload + full response, capture as much as possible from this list. **Evidence further down the list has more inference; evidence higher up is closer to ground truth.**
|
||||
|
||||
### Tier 1 — Pre-send / post-receive logs (best partial evidence)
|
||||
|
||||
The system you're investigating builds a request, then sends it. If the build step logs the assembled request **before** transmission, that log is ground truth for everything except the wire-level bytes (TLS, headers added by HTTP library, etc.).
|
||||
|
||||
```bash
|
||||
# Maximize debug logging
|
||||
APP_DEBUG=1 APP_LOG_LEVEL=debug APP_LOG_FILE=/tmp/trace.log ./target -x "minimal valid input" 2>&1 | head -200
|
||||
```
|
||||
|
||||
Look for log lines like:
|
||||
- `Building request: model=X, params={...}`
|
||||
- `[provider] payload: {...}`
|
||||
- `Sending to <url>: <serialized body>`
|
||||
|
||||
**Strength**: 95% of ground truth. Missing only wire-level transformations.
|
||||
|
||||
### Tier 2 — Local interception via proxy / shim
|
||||
|
||||
Run the real binary against a local proxy that records and (optionally) returns a canned response.
|
||||
|
||||
```bash
|
||||
# mitmproxy approach
|
||||
mitmproxy --listen-host 127.0.0.1 --listen-port 8888 --mode regular &
|
||||
HTTPS_PROXY=http://127.0.0.1:8888 SSL_CERT_FILE=~/.mitmproxy/mitmproxy-ca-cert.pem ./target ...
|
||||
# Now mitmproxy logs the actual TLS-decrypted request
|
||||
```
|
||||
|
||||
```bash
|
||||
# DYLD_INSERT_LIBRARIES / LD_PRELOAD shim approach
|
||||
# Wrap the network call to log payload, return a fake 200
|
||||
# See pwntools.md for shim examples
|
||||
```
|
||||
|
||||
**Strength**: Wire-level ground truth, but requires the target to honor your proxy / preload.
|
||||
|
||||
### Tier 3 — Static extraction × runtime fingerprint cross-check
|
||||
|
||||
When you cannot send a request at all, you can still cross-check static analysis with whatever the binary does that *doesn't* require the real call:
|
||||
|
||||
- The binary builds the request — even if sending fails, the build step ran. Trace it (Tier 1).
|
||||
- The binary writes a state file or cache — read it.
|
||||
- The binary emits version-specific User-Agent strings; verify they match your static extraction.
|
||||
- The binary's `--help` or `--version` output reveals build metadata; verify model lists / feature flags.
|
||||
|
||||
**Strength**: Disjoint evidence sources confirming the same fact. Two independent partial signals that agree are nearly as strong as one full observation.
|
||||
|
||||
### Tier 4 — Contrastive runtime under different inputs
|
||||
|
||||
If you can run with input variant A but not B, run A and reason about B from code:
|
||||
|
||||
```bash
|
||||
# A: minimal trial input — works for free tier
|
||||
./target --action=read --resource=local-file
|
||||
# B: full inference call — paid tier required, blocked
|
||||
# But the request-building code is shared between A and B!
|
||||
# Capture A's logs, then inspect the code path for B and verify only the model/endpoint diff.
|
||||
```
|
||||
|
||||
**Strength**: Confirms shared code paths; remaining gap is only the difference between A and B.
|
||||
|
||||
### Tier 5 — Vendor-published API logs / dashboard
|
||||
|
||||
If the operation succeeded earlier (before quota ran out, before access was revoked), the vendor's dashboard / audit log may show the request. Lower fidelity but still observed behavior.
|
||||
|
||||
**Strength**: Real wire data, but often summarized — token counts, status codes, no payload bodies.
|
||||
|
||||
### Tier 6 — Pure code reading with peer review
|
||||
|
||||
If literally none of the above is available, read the code carefully and submit it to **one Oracle for skeptical review** (see "Verification Oracle" below). This is the weakest tier and you must explicitly mark conclusions as "unverified" in the journal.
|
||||
|
||||
---
|
||||
|
||||
## How to combine partial signals
|
||||
|
||||
A defensible conclusion **prefers two independent signals from different tiers**, with one exception: a complete Tier 2 wire-level capture is wire-level ground truth and can stand alone for request-shape claims (because the wire bytes are exactly what the remote received). For *behavioral* claims (what the system does next, what state it stores, what side effects it produces), still combine with another signal.
|
||||
|
||||
| Available evidence | Defensibility |
|
||||
|---|---|
|
||||
| Tier 1 + Tier 1 (same log, different lines) | weak — single source |
|
||||
| Tier 1 + Tier 2 (debug log + proxy capture) | **strong** — independent confirmation |
|
||||
| Tier 1 + Tier 3 (debug log + version output cross-check) | **strong** — disjoint sources |
|
||||
| Tier 2 alone (full proxy capture) | strong **for request-shape claims only** — stands alone for "what bytes were sent". Add a second signal for response-handling or state claims. |
|
||||
| Tier 3 + Tier 4 (cross-check + contrastive run) | medium — both partial |
|
||||
| Tier 6 alone (code reading only) | **insufficient** — escalate or mark unverified |
|
||||
|
||||
Record in the journal:
|
||||
|
||||
```markdown
|
||||
## Partial runtime evidence
|
||||
### Question being verified
|
||||
<the specific claim, e.g. "Opus 4.7 default effort is 'high'">
|
||||
|
||||
### Available signals
|
||||
- Tier 1: debug log /tmp/trace.log line 47-49 shows `effort: "high"` ✓
|
||||
- Tier 3: static extraction of m5T() function returns "high" for smart mode ✓
|
||||
- Tier 6: code path verified by reading prompt-builder.js ✓
|
||||
|
||||
### Independence assessment
|
||||
Tier 1 and Tier 3 are independent — the log was emitted by a different
|
||||
code path than m5T() and would diverge if the static reading were wrong.
|
||||
|
||||
### Conclusion
|
||||
VERIFIED via Tier 1 + Tier 3 agreement. No need to escalate.
|
||||
```
|
||||
|
||||
If you cannot achieve a complete Tier 2 capture **or** two independent non-Tier-6 signals from the table above, **write an explicit note in the deliverable**:
|
||||
|
||||
> ⚠️ Partial-evidence finding. The full outbound payload could not be captured because [reason]. The conclusion rests on:
|
||||
> - [signal A — tier and source]
|
||||
> - [signal B — tier and source]
|
||||
> A future verification should attempt [the missing tier] when [condition].
|
||||
|
||||
---
|
||||
|
||||
## Verification Oracle pattern (for non-debug tasks)
|
||||
|
||||
The skill's main Oracle Triple (`04-oracle-triple.md`) is for **stuck debugging** — 2 failed rounds, mental box, three orthogonal framings to break out.
|
||||
|
||||
For tasks where the deliverable is an **artifact, not a bug fix** (reverse engineering, extraction, audit, compliance documentation), use a different pattern: **single Oracle, late, skeptical, with the deliverable in hand**.
|
||||
|
||||
### When to invoke
|
||||
|
||||
- Right before declaring an extraction/audit task "done"
|
||||
- After every significant revision of the deliverable (not after every small edit)
|
||||
- Maximum 3-4 iterations before escalating to user
|
||||
|
||||
### Pattern
|
||||
|
||||
```
|
||||
task(subagent_type="oracle", load_skills=[], run_in_background=false,
|
||||
prompt="""
|
||||
SKEPTICAL FINAL VERIFICATION — be critical, look for reasons the task is incomplete or wrong.
|
||||
|
||||
## Original task
|
||||
<verbatim user request>
|
||||
|
||||
## What I produced
|
||||
<list of artifacts with paths and brief descriptions>
|
||||
|
||||
## Specific claims to verify
|
||||
<bullet list of every concrete claim in the deliverable>
|
||||
|
||||
## Where to look
|
||||
<paths the Oracle should Read / Bash to verify>
|
||||
|
||||
## Your job
|
||||
1. Read the deliverables.
|
||||
2. Spot-check each claim against the source/evidence the deliverable cites.
|
||||
3. Identify any unsubstantiated claims, missing pieces, or factual errors.
|
||||
4. End with PASS / FAIL / PARTIAL with specific gaps.
|
||||
Be skeptical. Don't rubber-stamp.
|
||||
""")
|
||||
```
|
||||
|
||||
### Why this differs from the Oracle Triple
|
||||
|
||||
| | Oracle Triple (debug) | Verification Oracle (artifact) |
|
||||
|---|---|---|
|
||||
| Trigger | 2 failed hypothesis rounds | About to declare "done" |
|
||||
| Count | 3 in parallel, orthogonal framings | 1 sequential, focused review |
|
||||
| Goal | Break out of mental box | Catch unsubstantiated claims |
|
||||
| Tone of prompt | Brainstorm wide alternatives | Skeptical audit |
|
||||
| Iteration | Reset hypothesis set after | Fix gaps, re-invoke until PASS |
|
||||
|
||||
### Don't conflate them
|
||||
|
||||
If you're stuck debugging, do the Triple. If you have a deliverable and need it audited, do the Verification Oracle. Doing the Triple on a finished extraction will return three diverging "what if you tried…" tangents that are not what you need. Doing the Verification Oracle on a stuck debugging session will return a polite "the evidence is incomplete" that you already knew.
|
||||
|
||||
---
|
||||
|
||||
## Common partial-evidence anti-patterns
|
||||
|
||||
| Anti-pattern | Why it fails | Replacement |
|
||||
|---|---|---|
|
||||
| "It looks right in the code, so it works" | Tier 6 alone, unverified | Add at least one Tier 1-3 signal |
|
||||
| "I ran it once, didn't error, so it's correct" | Absence of error ≠ presence of correctness | Capture the actual output and verify content |
|
||||
| "The mock returns the value I wrote, so the code is fine" | Tautology — mock loops back your assumption | Use Tier 2 (proxy) instead, or cross-check with Tier 3 |
|
||||
| "The vendor's dashboard shows my call worked" | Dashboard often only shows status code, not behavior | Combine with Tier 1 if available |
|
||||
| "I'll trust the most-recent stack overflow answer" | Code from a different version / context | Verify against the actual binary you have |
|
||||
|
||||
---
|
||||
|
||||
## Cleanup additions for partial-evidence work
|
||||
|
||||
```bash
|
||||
# Proxy artifacts
|
||||
pkill -f mitmproxy 2>/dev/null
|
||||
rm -f ~/.mitmproxy/cache_* 2>/dev/null
|
||||
|
||||
# Debug log files
|
||||
rm -f /tmp/trace.log /tmp/*-debug-trace.log
|
||||
|
||||
# DYLD_INSERT / LD_PRELOAD shim libraries
|
||||
rm -f /tmp/*.dylib /tmp/*.so
|
||||
|
||||
# Verify env vars set in your shell are not persisted
|
||||
unset HTTPS_PROXY APP_DEBUG APP_LOG_LEVEL APP_LOG_FILE 2>/dev/null
|
||||
```
|
||||
@@ -0,0 +1,415 @@
|
||||
# Bundled-JS / Embedded-Source Binaries (Bun SEA, Node SEA, Deno compile, pkg, Electron, PyInstaller)
|
||||
|
||||
A growing class of "binaries" are not stripped C/C++ at all — they are a runtime VM glued onto a high-level-language bundle. The bundle is **plaintext or trivially-decodable** inside the binary.
|
||||
|
||||
If you reach for `native-binary.md` workflow on these (Ghidra → pwndbg → hex), you will waste hours decompiling a runtime you don't care about while the actual logic sits exposed three megabytes away.
|
||||
|
||||
**This reference exists because the workflow is fundamentally different from stripped C.**
|
||||
|
||||
---
|
||||
|
||||
## When to use this reference instead of `native-binary.md`
|
||||
|
||||
Open this if `file ./target` shows a generic Mach-O / ELF / PE BUT any of:
|
||||
|
||||
- Size is suspiciously large (50 MB+ for a "simple CLI")
|
||||
- `strings -n 8 ./target | rg -i "node_modules|webpack|esbuild|bun|pkg/lib|electron|pyinstaller"` returns hits
|
||||
- The binary's CLI flags include things like `--inspect`, `--unhandled-rejections`, npm-style help text
|
||||
- Vendor docs say it's built with Bun / pkg / nexe / Deno compile / PyInstaller / Electron / Tauri (UI shell)
|
||||
- `head -c 4 ./target | xxd` shows a known runtime magic for an embedded archive section
|
||||
|
||||
If yes → **stop following `native-binary.md` and follow this**. Triage and dynamic tracing are the same. Static analysis is completely different.
|
||||
|
||||
---
|
||||
|
||||
## The workflow
|
||||
|
||||
```
|
||||
[1] Triage → identify the bundler (Bun? pkg? Deno? Electron? PyInstaller?)
|
||||
[2] Locate the bundle → find where the embedded source archive starts
|
||||
[3] Extract → dump source to disk so you can grep / read it
|
||||
[4] Source-level static analysis (rg + Read, NOT Ghidra)
|
||||
[5] Runtime verification → debug logs, --inspect, partial-evidence patterns
|
||||
[6] Fix / report
|
||||
```
|
||||
|
||||
Step 3 is the unlock — once you have plaintext source on disk, the rest is normal codebase exploration.
|
||||
|
||||
---
|
||||
|
||||
## [1] Identify the bundler — 30-second fingerprint
|
||||
|
||||
```bash
|
||||
# Look for runtime-specific markers in plaintext strings
|
||||
strings -n 12 ./target 2>/dev/null | rg -iE 'bun|node_modules|webpack|esbuild|deno|pkg/lib|electron|pyinstaller|nexe|NODE_SEA_FUSE|NODE_SEA_BLOB|tauri|ESZIP_V2|denort' | head -20
|
||||
```
|
||||
|
||||
| Marker pattern | Bundler | Source format |
|
||||
|---|---|---|
|
||||
| `@oven/bun-darwin`, `bun-lockfile-format-v`, `// @bun` | **Bun SEA** (compiled via `bun build --compile`) | Plaintext JS, single big bundle |
|
||||
| `NODE_SEA_BLOB` + `NODE_SEA_FUSE_fce680ab2cc467b6e072b8b5df1996b2:1` | **Node SEA** (`node --build-sea` or `--experimental-sea-config`) | Plaintext JS, or V8 code cache (when `useCodeCache: true`), or startup snapshot (when `useSnapshot: true`) — the latter two are NOT plaintext |
|
||||
| `pkg/lib/bootstrap.js`, `pkg/prelude`, `PAYLOAD_POSITION` | **pkg** (vercel/pkg) | Plaintext or v8 cached data |
|
||||
| `ESZIP_V2`, `denort`, `deno_runtime` | **Deno compile** (`deno compile`) | TS/JS in eszip archive — readable but needs `eszip` crate to walk; not pure plaintext |
|
||||
| `Electron`, `app.asar`, `chrome.dll`, `Squirrel.Mac` | **Electron** | `app.asar` archive (TAR-like with JSON header). Source is plaintext JS once extracted |
|
||||
| `PyInstaller`, `pyz`, `_MEIPASS`, `pyi-os-utils` | **PyInstaller** | Compressed `.pyc` bytecode — needs `pyinstxtractor` + `decompyle3` to recover Python source |
|
||||
| `nexe-`, `nexe_compile`, `:::nexe::` | **nexe** | Plaintext JS appended to node binary |
|
||||
| `Tauri`, `tao`, `wry`, `tauri::generate_context` | **Tauri** (Rust shell + JS UI) | **Two worlds**: JS frontend in resource section is extractable here; Rust commands / core logic are native and require [native-binary.md](native-binary.md) |
|
||||
|
||||
If multiple match (e.g. Tauri + Bun): the outer shell is the first one (Tauri/Electron). The inner JS is the second one's format. **For Tauri specifically, expect to use both this reference (for the UI bundle) and `native-binary.md` (for the Rust binary side).**
|
||||
|
||||
> **Source-format reality check**: only Bun SEA, pkg (when not using `--public-packages`), nexe, and Electron `.asar` are reliably plaintext. Node SEA with code-cache or snapshot, PyInstaller `.pyc`, and Deno eszip require additional tooling. Don't assume `strings` will find readable code — verify the bundler first.
|
||||
|
||||
---
|
||||
|
||||
## [2] Locate the bundle
|
||||
|
||||
### Bun SEA — JS is just embedded plaintext
|
||||
|
||||
The JS source is concatenated into the binary as a giant template literal / string. No decoding needed.
|
||||
|
||||
```bash
|
||||
# Verify by searching for typical JS bundle markers
|
||||
strings -n 8 ./target | rg "function|var |let |const |async function" | head -5
|
||||
|
||||
# Find where the bundle starts (look for "use strict" or banner comment)
|
||||
LC_ALL=C grep -aob '"use strict"' ./target | head -5
|
||||
LC_ALL=C grep -aob '#!/usr/bin/env bun' ./target | head -5
|
||||
```
|
||||
|
||||
### Node SEA — `NODE_SEA_BLOB` resource/segment + activated fuse
|
||||
|
||||
Per the [Node.js SEA docs](https://nodejs.org/api/single-executable-applications.html), a Node-built SEA contains:
|
||||
- A resource (PE), section in `NODE_SEA` segment (Mach-O), or note (ELF) named `NODE_SEA_BLOB`
|
||||
- The fuse string `NODE_SEA_FUSE_fce680ab2cc467b6e072b8b5df1996b2:1` (with trailing `:1` indicating injected; `:0` means a copy of the node binary that has not yet had a blob injected)
|
||||
|
||||
```bash
|
||||
# Confirm it is a SEA at all
|
||||
LC_ALL=C grep -aob 'NODE_SEA_FUSE_fce680ab2cc467b6e072b8b5df1996b2:1' ./target | head -1
|
||||
|
||||
# Find the blob resource/section
|
||||
LC_ALL=C grep -aob 'NODE_SEA_BLOB' ./target | head
|
||||
|
||||
# On Mach-O, inspect the segment directly
|
||||
otool -l ./target | grep -A4 'NODE_SEA'
|
||||
```
|
||||
|
||||
The blob format is documented but non-trivial to walk by hand. For extraction, **use postject in reverse** (carve the section bytes) or read the blob via `node:sea` API from inside a debug build of the same binary. Plain `strings` will get you the embedded JS only when the SEA was built without `useCodeCache` and without `useSnapshot` — both of those replace plaintext with V8 cache data or startup snapshot bytes.
|
||||
|
||||
`node --build-sea sea-config.json` and `node --experimental-sea-config sea-config.json` *generate* SEA blobs; neither inspects an existing executable.
|
||||
|
||||
### Deno compile — eszip archive section
|
||||
|
||||
```bash
|
||||
# Deno-compile binaries embed an eszip v2 archive
|
||||
LC_ALL=C grep -aob 'ESZIP_V2' ./target | head -3
|
||||
# Also confirm the runtime
|
||||
LC_ALL=C grep -aob 'denort' ./target | head -1
|
||||
```
|
||||
|
||||
To extract, use the `eszip` Rust crate (or the `@deno/eszip` JS port) to parse the archive after carving it out at the offset above. There is no stable Deno CLI flag that inspects compiled-executable eszip contents as of 2026-04 — `deno info` only works on source files.
|
||||
|
||||
### pkg — `PAYLOAD_POSITION` marker
|
||||
|
||||
```bash
|
||||
LC_ALL=C grep -aob 'PAYLOAD_POSITION' ./target | head
|
||||
LC_ALL=C grep -aob 'pkg/prelude' ./target | head
|
||||
```
|
||||
|
||||
For source extraction, use the `pkg-extract` tooling community projects or carve based on the offset reported by the `PAYLOAD_POSITION:<n>` value.
|
||||
|
||||
### Electron — `app.asar` is usually a separate file
|
||||
|
||||
Most Electron apps ship `app.asar` next to the binary, not embedded inside. Extract it with the official tool:
|
||||
|
||||
```bash
|
||||
# macOS layout
|
||||
ls -la /Applications/MyApp.app/Contents/Resources/app.asar
|
||||
npx @electron/asar extract app.asar ./extracted/
|
||||
# or older:
|
||||
npx asar extract app.asar ./extracted/
|
||||
```
|
||||
|
||||
For single-file builds where the asar is embedded inside the executable, **do not pattern-match arbitrary 4-byte sequences** (the asar format starts with a Pickle-encoded uint32 header size + JSON metadata, and the same bytes appear elsewhere in any binary). Instead, use a Pickle-aware extractor that validates the JSON header before claiming a match — the `asar` npm package's programmatic `extractAll()` API does this. Carve the asar bytes by scanning for a candidate Pickle header (4-byte size + 4-byte payload size + `{"files":` prefix), validate the JSON parses, then feed the carved buffer to `extractAll()`.
|
||||
|
||||
### PyInstaller — use `pyinstxtractor`, NOT runtime self-extraction
|
||||
|
||||
```bash
|
||||
# Recover the embedded archive without running the binary
|
||||
python3 pyinstxtractor.py ./target
|
||||
# Output: ./target_extracted/ with .pyc files
|
||||
|
||||
# Decompile the .pyc files back to Python source
|
||||
decompyle3 ./target_extracted/main.pyc # Python 3.7+
|
||||
uncompyle6 ./target_extracted/main.pyc # older Python
|
||||
```
|
||||
|
||||
If `pyinstxtractor` cannot read the archive (e.g. non-standard PyInstaller version), use the official `pyi-archive_viewer` tool that ships with PyInstaller. Avoid the "run-the-binary-and-snoop-`/tmp/_MEI*`" approach: it only catches what runs in the time window between `_MEIPASS` extraction and cleanup, and it executes potentially untrusted code.
|
||||
|
||||
---
|
||||
|
||||
## [3] Extract source to disk — DO NOT skip this
|
||||
|
||||
**The single biggest mistake** with bundled-JS reverse engineering is trying to read the source out of `strings` output or `xxd` dumps. You will lose data. See "Gotchas" below.
|
||||
|
||||
### For Bun SEA / nexe / single-string-blob bundlers
|
||||
|
||||
Read the binary as bytes, find the JS section, save to a `.js` file:
|
||||
|
||||
```python
|
||||
# extract_bundled_js.py
|
||||
import sys
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
raise SystemExit("usage: extract_bundled_js.py <target>")
|
||||
|
||||
with open(sys.argv[1], 'rb') as f:
|
||||
data = f.read()
|
||||
|
||||
markers = [b'// @bun', b'"use strict"', b"'use strict'", b'#!/usr/bin/env']
|
||||
start = -1
|
||||
for m in markers:
|
||||
p = data.find(m)
|
||||
if p != -1 and (start == -1 or p < start):
|
||||
start = p
|
||||
|
||||
if start == -1:
|
||||
raise SystemExit(
|
||||
"no bundle marker found — binary may not be Bun/nexe, "
|
||||
"or markers were stripped. Try strings(1) for hints."
|
||||
)
|
||||
|
||||
# Heuristic end: look for a long null run AFTER start.
|
||||
# This is a heuristic, NOT a guarantee. Verify the tail of the output
|
||||
# looks like JS (closing braces, EOF) before trusting it.
|
||||
end = data.find(b'\x00' * 1024, start)
|
||||
if end == -1:
|
||||
end = len(data)
|
||||
|
||||
bundle = data[start:end]
|
||||
print(f'Extracted {len(bundle)} bytes from offset {start} to {end}', file=sys.stderr)
|
||||
sys.stdout.buffer.write(bundle)
|
||||
```
|
||||
|
||||
```bash
|
||||
python3 extract_bundled_js.py ./target > extracted-bundle.js
|
||||
wc -c extracted-bundle.js
|
||||
# Sanity check the tail is JS, not random binary
|
||||
tail -c 200 extracted-bundle.js
|
||||
```
|
||||
|
||||
### For PyInstaller
|
||||
|
||||
Use `pyinstxtractor` then `uncompyle6` / `decompyle3` on the `.pyc` files.
|
||||
|
||||
### For Electron .asar
|
||||
|
||||
```bash
|
||||
npx asar extract app.asar ./extracted/
|
||||
# Now ./extracted/ has a normal node_modules + your source layout
|
||||
```
|
||||
|
||||
### For Deno compile
|
||||
|
||||
Use the `eszip` Rust crate or the `@deno/eszip` JS port to walk the archive after carving the eszip section out at the offset reported by the `ESZIP_V2` magic search. There is no stable Deno CLI as of 2026-04 that inspects compiled-binary eszip contents directly.
|
||||
|
||||
---
|
||||
|
||||
## [4] Source-level static analysis — `rg` + `Read`, not Ghidra
|
||||
|
||||
Once you have the source on disk, treat it as a normal codebase:
|
||||
|
||||
```bash
|
||||
# Find function definitions
|
||||
rg -n "^function |^const \w+ = (function|\(.*\) =>)" extracted-bundle.js | head
|
||||
|
||||
# Find specific behavior
|
||||
rg -n "claude-opus-4-7|reasoning_effort|api_key" extracted-bundle.js
|
||||
|
||||
# Resolve minified identifiers — they show up as `var XYZ="value"`
|
||||
rg -aoP 'var \w+="[^"]+"' extracted-bundle.js | head -50
|
||||
```
|
||||
|
||||
For minified bundles, use a template-literal-aware parser to extract specific functions or template strings. Example skeleton:
|
||||
|
||||
```python
|
||||
def find_template_end(data, start):
|
||||
"""Walk a JS template literal preserving ${...} interpolation depth.
|
||||
Returns position of closing backtick."""
|
||||
i = start
|
||||
while i < len(data):
|
||||
c = data[i:i+1]
|
||||
if c == b'\\':
|
||||
i += 2; continue
|
||||
if c == b'$' and data[i+1:i+2] == b'{':
|
||||
depth = 1; i += 2
|
||||
while i < len(data) and depth > 0:
|
||||
cc = data[i:i+1]
|
||||
if cc == b'\\': i += 2; continue
|
||||
if cc == b'`':
|
||||
j = find_template_end(data, i+1)
|
||||
i = j + 1; continue
|
||||
if cc == b'{': depth += 1
|
||||
elif cc == b'}': depth -= 1
|
||||
elif cc in (b'"', b"'"):
|
||||
q = cc; i += 1
|
||||
while i < len(data) and data[i:i+1] != q:
|
||||
if data[i:i+1] == b'\\': i += 2
|
||||
else: i += 1
|
||||
i += 1; continue
|
||||
i += 1
|
||||
continue
|
||||
if c == b'`': return i
|
||||
i += 1
|
||||
return -1
|
||||
```
|
||||
|
||||
For function-body extraction, **track the parameter list separately** before tracking body braces. The naive approach mis-counts destructuring `function f({a, b, ...c})` as the body `{` and exits early.
|
||||
|
||||
---
|
||||
|
||||
## [5] Runtime verification
|
||||
|
||||
You usually cannot single-step JS inside a Bun-compiled binary the way you would with `node --inspect`. Workarounds:
|
||||
|
||||
### Bun-compiled
|
||||
|
||||
Bun's inspector takes `--inspect[=<host>:<port>[/<prefix>]]` on the command line. For env-var control of compiled binaries, the form is the same minus the leading `--`:
|
||||
|
||||
```bash
|
||||
# Default port (6499) auto-prefix
|
||||
./target --inspect
|
||||
# → ws://localhost:6499/<auto-prefix> (paste into https://debug.bun.sh)
|
||||
|
||||
# Explicit host:port[/prefix]
|
||||
./target --inspect=localhost:9229/dbg
|
||||
# Or via env var (if --inspect cannot be passed)
|
||||
BUN_INSPECT=localhost:9229/dbg ./target
|
||||
```
|
||||
|
||||
**For HTTP request tracing without an interactive debugger** (highest-value Bun-specific runtime evidence):
|
||||
|
||||
```bash
|
||||
# Print every fetch() / node:http request as a curl command + full headers/body
|
||||
BUN_CONFIG_VERBOSE_FETCH=curl ./target ...
|
||||
|
||||
# Or just print the request/response without curl-format
|
||||
BUN_CONFIG_VERBOSE_FETCH=true ./target ...
|
||||
```
|
||||
|
||||
Plus generic env-var-based debug logging if the app supports it:
|
||||
|
||||
```bash
|
||||
APP_DEBUG=1 APP_LOG_LEVEL=debug APP_LOG_FILE=/tmp/trace.log ./target
|
||||
```
|
||||
|
||||
### Node SEA / pkg / nexe
|
||||
```bash
|
||||
# These usually accept --inspect since they are real Node
|
||||
./target --inspect
|
||||
# Then chrome://inspect or node --inspect-brk
|
||||
```
|
||||
|
||||
### Electron
|
||||
```bash
|
||||
./target.app/Contents/MacOS/target --inspect=9229 --remote-debugging-port=9223
|
||||
# Renderer process is at chrome://inspect, main process via the inspector port
|
||||
```
|
||||
|
||||
### When you cannot make a real call
|
||||
The target's API may require credentials, network access, or paid quota you don't have. **You are not stuck** — see [methodology/partial-runtime-evidence.md](../methodology/partial-runtime-evidence.md) for the fallback patterns.
|
||||
|
||||
---
|
||||
|
||||
## ⚠️ Gotchas — read these before extracting
|
||||
|
||||
### G1. `strings -n N` silently drops short identifier interpolations
|
||||
|
||||
`strings` outputs runs of printable characters of length **≥ N**. Default is 4 on most systems; many references (including older versions of `native-binary.md`) recommend `-n 8` for less noise.
|
||||
|
||||
**With `-n 8`, short template-literal interpolations like `${x}`, `${i}`, `${R}` are silently dropped** because they are 4 chars surrounded by non-printable bytes (newlines or section padding). The result looks like:
|
||||
|
||||
```text
|
||||
expected: <INSTRUCTIONS>\n${x}\n</INSTRUCTIONS>
|
||||
strings: <INSTRUCTIONS>\n</INSTRUCTIONS> ← ${x} is gone, no warning
|
||||
```
|
||||
|
||||
A consumer reading the strings output would conclude the template is empty.
|
||||
|
||||
**Mitigation**:
|
||||
1. Use `strings` only for **fingerprinting** (Phase 1 triage), never as the source of extracted text.
|
||||
2. For actual extraction, **read the binary as bytes** with `python3 -c "open('./target','rb').read()"` and grep / parse from there.
|
||||
3. If you must use `strings`, try `strings -n 1 -t x ./target` and post-filter — but byte-level reads are still more reliable.
|
||||
|
||||
### G2. Stale cached binary ≠ latest features
|
||||
|
||||
Bundled-app installers often check a remote version and skip download if a cached binary exists. If you reverse-engineered an old version and the user reports behavior you don't see in the source, **re-run the installer** (or fetch the version manifest manually) before assuming the source is current.
|
||||
|
||||
```bash
|
||||
# Example pattern - varies by tool
|
||||
curl -fsSL https://example.com/install.sh | head -50 # find version-fetch URL
|
||||
curl -fsSL https://static.example.com/cli/cli-version.txt
|
||||
./your-tool --version
|
||||
# Compare. If different, re-install.
|
||||
```
|
||||
|
||||
### G3. APFS / NTFS case-insensitivity silently overwrites files
|
||||
|
||||
When extracting many minified function bodies (`cVR`, `CVR`, `dpr`, `DPR`, …) and saving each to its own file, **macOS APFS and Windows NTFS treat `cVR.txt` and `CVR.txt` as the same file**. The second write silently overwrites the first.
|
||||
|
||||
**Mitigation**: prefix filenames with something case-distinguishing, e.g. `mode-cVR.txt`, `mode-CVR.txt`, or use a hash suffix.
|
||||
|
||||
### G4. Bun's runtime adds 30-50 MB of unrelated symbols
|
||||
|
||||
A 70 MB Bun-compiled binary is **mostly Bun runtime** (~50 MB) plus your app (~20 MB). When fingerprinting, you will see thousands of strings like `tree-sitter-typescript`, `react-native-stylex` etc. that the user's actual app doesn't use — these are package names baked into Bun's package-resolution data.
|
||||
|
||||
**Mitigation**: when grepping for "what does this app do?", filter out runtime noise:
|
||||
```bash
|
||||
strings -n 8 ./target | rg -v 'node_modules|@oven/bun|package-lock|tree-sitter|ffmpeg-installer' | head
|
||||
```
|
||||
|
||||
### G5. Source maps usually NOT shipped
|
||||
|
||||
Bundled apps strip source maps for production. Variable names are minified to `T`, `R`, `a`, `r`, etc. Treat the bundle like an obfuscated codebase: identify constants by tracing assignments (`var T="actual-name"`) and resolve interpolations manually.
|
||||
|
||||
### G6. The "extract" file is not legally redistributable
|
||||
|
||||
If reverse-engineering proprietary software, the extracted source is the vendor's IP. Use it for understanding behavior, **never commit it to git**, never post snippets in public issues. Cleanup your `extracted-bundle.js` files in Phase 9.
|
||||
|
||||
---
|
||||
|
||||
## Silent-failure patterns specific to bundled JS
|
||||
|
||||
| Pattern | Why it's silent |
|
||||
|---|---|
|
||||
| Bundle includes unreachable dead code from tree-shaking failures | You read code that never runs — verify with runtime trace |
|
||||
| `process.env.X` resolved at BUILD time, not RUNTIME | Setting the env var at runtime has no effect; the value is baked in |
|
||||
| `import.meta.url` in compiled binary returns `bun://...` not a real path | File-relative resolution silently breaks |
|
||||
| Worker threads spawn from embedded code, look for sub-bundle inside main bundle | Workers may have their own copy of dependencies |
|
||||
| Minified identifiers with case variants used in same module | Easy to confuse `cVR` with `CVR` when reading fast |
|
||||
|
||||
---
|
||||
|
||||
## Phase 9 cleanup specifics for bundled-JS work
|
||||
|
||||
```bash
|
||||
# Remove extracted bundles — they may contain proprietary source
|
||||
rm -f /tmp/extracted-bundle.js /tmp/extracted-*.js
|
||||
rm -rf /tmp/asar-extracted/
|
||||
rm -rf /tmp/_MEI*
|
||||
|
||||
# Remove strings dumps
|
||||
rm -f /tmp/*-strings.txt /tmp/*-strings-v*.txt
|
||||
|
||||
# Remove Python helper scripts created for parsing
|
||||
rm -f /tmp/extract_bundled_js.py /tmp/parse_template.py
|
||||
|
||||
# Verify the extraction directory is gone (if you used a workspace dir)
|
||||
ls /Users/$USER/local-workspaces/*-extracted/ 2>/dev/null
|
||||
# rm -rf only after journal review confirms nothing important is there
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## When to escalate back to `native-binary.md`
|
||||
|
||||
If extraction reveals the "bundle" is actually compiled to v8 cached data (pkg with `--public-packages` or PyInstaller with bytecode-only mode), and decompilation is non-trivial, **switch back to `native-binary.md` workflow** (Ghidra against the runtime + careful tracing). Bundled-JS workflow only helps when the high-level source is recoverable as readable text.
|
||||
@@ -0,0 +1,252 @@
|
||||
# Go Debugging
|
||||
|
||||
Covers goroutines, `dlv` (Delve), `pprof`, the race detector, and the fact that Go's concurrency model means most bugs are about goroutines doing something quiet and wrong.
|
||||
|
||||
---
|
||||
|
||||
## Environment detection (Phase 0)
|
||||
|
||||
```bash
|
||||
go version
|
||||
cat go.mod | head -5
|
||||
|
||||
# Delve installed?
|
||||
which dlv
|
||||
dlv version
|
||||
|
||||
# Build constraints
|
||||
grep -r '// +build\|//go:build' cmd/ internal/ pkg/ 2>/dev/null | head
|
||||
|
||||
# pprof wired up?
|
||||
grep -r 'net/http/pprof\|runtime/pprof' --include='*.go' | head -3
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Delve (`dlv`) — the Go debugger
|
||||
|
||||
Go's gc compiler emits DWARF, but plain gdb barely understands goroutines. **Use dlv, not gdb.** Plain gdb on a Go binary will miss goroutine state and print garbage for interface values.
|
||||
|
||||
### The five `dlv` launch modes
|
||||
|
||||
```bash
|
||||
# Build and launch under debugger (equivalent to `go run` + debug)
|
||||
dlv debug ./cmd/server -- --port=8080
|
||||
|
||||
# Debug a test binary
|
||||
dlv test ./internal/handler/ # enters the test package under debug
|
||||
|
||||
# Debug an existing binary (must be built with -gcflags="all=-N -l" for best results)
|
||||
dlv exec ./bin/myserver
|
||||
|
||||
# Attach to a running process
|
||||
dlv attach $(pgrep myserver)
|
||||
|
||||
# Headless mode (IDE / remote attach) — default port 2345
|
||||
dlv debug --headless --listen=:2345 --api-version=2 ./cmd/server
|
||||
```
|
||||
|
||||
### Building a debuggable binary
|
||||
|
||||
The compiler inlines and optimizes aggressively in normal builds, which makes stepping confusing. For serious debugging:
|
||||
|
||||
```bash
|
||||
go build -gcflags="all=-N -l" -o ./bin/server ./cmd/server
|
||||
# -N disables optimization
|
||||
# -l disables inlining
|
||||
```
|
||||
|
||||
Then `dlv exec ./bin/server`.
|
||||
|
||||
### Essential dlv commands
|
||||
|
||||
```
|
||||
(dlv) b main.main # breakpoint at function
|
||||
(dlv) b handler.go:42 # breakpoint at file:line
|
||||
(dlv) b pkg/foo.Bar # breakpoint at type method (Go path syntax)
|
||||
(dlv) c / continue # continue until next break
|
||||
(dlv) n / next # step over
|
||||
(dlv) s / step # step into
|
||||
(dlv) so / stepout # step out
|
||||
(dlv) bt / stack # stack trace of current goroutine
|
||||
(dlv) goroutines # list all goroutines
|
||||
(dlv) goroutine <id> # switch to goroutine N
|
||||
(dlv) goroutine <id> bt # stack of a specific goroutine
|
||||
(dlv) locals # all locals in frame
|
||||
(dlv) args # function args
|
||||
(dlv) p <expr> # print value (understands interfaces, maps, slices)
|
||||
(dlv) vars <regex> # package vars matching regex
|
||||
(dlv) regs # registers (rare in Go debugging)
|
||||
(dlv) on <bpid> print <expr> # auto-print on breakpoint hit (powerful!)
|
||||
(dlv) trace <location> # like breakpoint but just logs, doesn't stop
|
||||
```
|
||||
|
||||
The `trace` command is underused — it's like a logpoint, no stepping required.
|
||||
|
||||
---
|
||||
|
||||
## Goroutine-centric debugging
|
||||
|
||||
Goroutine leaks and deadlocks are the most common Go bugs. `dlv`'s `goroutines` command is the starting point.
|
||||
|
||||
```
|
||||
(dlv) goroutines -t # with truncated stack
|
||||
(dlv) goroutines -s # sorted by stack
|
||||
(dlv) goroutines -with user # filter user-spawned goroutines
|
||||
```
|
||||
|
||||
Common patterns:
|
||||
|
||||
| You see in `goroutines` | Usually means |
|
||||
|---|---|
|
||||
| 100s of goroutines stuck at `chan receive` | Producer died; consumers leak |
|
||||
| 100s stuck at `semacquire` | Lock contention; a holder probably deadlocked |
|
||||
| One stuck at `select` with no default | Missing case or closed channel scenario |
|
||||
| Stuck at `netpoll` | External I/O not responding — not a Go bug, check downstream |
|
||||
| Growing count over time | Goroutine leak — need to find who's spawning without cleanup |
|
||||
|
||||
### Panic signals in Go
|
||||
|
||||
```go
|
||||
// Without recovery, panics crash the program with a stack trace of ALL goroutines
|
||||
// With recovery, they're silent unless explicitly logged:
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("recovered panic: %v\n%s", r, debug.Stack()) // GOOD
|
||||
// log.Printf("recovered") // BAD — silent
|
||||
}
|
||||
}()
|
||||
```
|
||||
|
||||
**Always check for silent recovers** in Phase 8. Grep:
|
||||
```bash
|
||||
rg 'recover\(\)' --type go
|
||||
```
|
||||
|
||||
And inspect each site for whether the panic is actually surfaced.
|
||||
|
||||
---
|
||||
|
||||
## Race detector — ALWAYS run when the bug is intermittent
|
||||
|
||||
```bash
|
||||
go test -race ./...
|
||||
go run -race ./cmd/server
|
||||
go build -race ./cmd/server
|
||||
```
|
||||
|
||||
The race detector wraps memory accesses and catches concurrent read/write without synchronization. **Run this before attaching dlv** if intermittency is involved — it often finds the bug directly.
|
||||
|
||||
Output shape:
|
||||
```
|
||||
WARNING: DATA RACE
|
||||
Read at 0x00c0001a0080 by goroutine 7:
|
||||
main.(*Counter).Value()
|
||||
/path/to/counter.go:14 +0x3c
|
||||
Previous write at 0x00c0001a0080 by goroutine 6:
|
||||
main.(*Counter).Inc()
|
||||
/path/to/counter.go:10 +0x5f
|
||||
```
|
||||
|
||||
Both stacks. Both goroutines. The race is obvious from the line pair.
|
||||
|
||||
---
|
||||
|
||||
## pprof — for perf, memory, goroutine leaks
|
||||
|
||||
### Wire it up (idempotent; usually already present)
|
||||
|
||||
```go
|
||||
import _ "net/http/pprof"
|
||||
|
||||
func main() {
|
||||
go func() {
|
||||
log.Println(http.ListenAndServe("localhost:6060", nil))
|
||||
}()
|
||||
// ... rest of your server
|
||||
}
|
||||
```
|
||||
|
||||
### Queries
|
||||
|
||||
```bash
|
||||
# CPU profile (30s)
|
||||
go tool pprof http://localhost:6060/debug/pprof/profile?seconds=30
|
||||
|
||||
# Heap snapshot
|
||||
go tool pprof http://localhost:6060/debug/pprof/heap
|
||||
|
||||
# Goroutine snapshot — find leaks
|
||||
go tool pprof http://localhost:6060/debug/pprof/goroutine
|
||||
|
||||
# Block profile — find blocking ops (needs runtime.SetBlockProfileRate)
|
||||
go tool pprof http://localhost:6060/debug/pprof/block
|
||||
|
||||
# Mutex profile — find lock contention (needs runtime.SetMutexProfileFraction)
|
||||
go tool pprof http://localhost:6060/debug/pprof/mutex
|
||||
```
|
||||
|
||||
Inside pprof:
|
||||
```
|
||||
(pprof) top # top functions by self time
|
||||
(pprof) list main.handler # annotated source of a function
|
||||
(pprof) web # SVG callgraph in browser (requires graphviz)
|
||||
(pprof) traces # sample traces
|
||||
```
|
||||
|
||||
For goroutine leaks, **take two snapshots 30s apart** and diff:
|
||||
```bash
|
||||
go tool pprof -base prof1.pb.gz prof2.pb.gz
|
||||
```
|
||||
|
||||
Goroutines that appear in prof2 but not prof1 are new; if they stick around, they're leaking.
|
||||
|
||||
---
|
||||
|
||||
## `GODEBUG` — runtime-level observability
|
||||
|
||||
```bash
|
||||
GODEBUG=gctrace=1 ./myserver # print GC stats
|
||||
GODEBUG=schedtrace=1000 ./myserver # scheduler trace every 1000ms
|
||||
GODEBUG=scheddetail=1,schedtrace=1000 # detailed scheduler state
|
||||
GODEBUG=allocfreetrace=1 ./myserver # every alloc/free (noisy!)
|
||||
GODEBUG=memprofilerate=1 ./myserver # profile every allocation
|
||||
```
|
||||
|
||||
Useful for diagnosing GC pressure, goroutine starvation, or memory pattern issues.
|
||||
|
||||
---
|
||||
|
||||
## Silent-failure patterns in Go
|
||||
|
||||
| Pattern | Why it's silent |
|
||||
|---|---|
|
||||
| `if err != nil { return err }` that returns to a caller that ignores | Error bubbles up, then gets discarded at the top |
|
||||
| `defer func() { recover() }()` — bare recover, no log | Panic swallowed, program continues with state corruption |
|
||||
| `_, _ = conn.Write(data)` | Intentionally discarded error |
|
||||
| Buffered channel send that blocks forever | Sender hangs; hard to see if no deadlock detection |
|
||||
| `time.Sleep` in a test | "Works on my machine"; test passes locally, fails in CI |
|
||||
| `go func() { ... }()` with no error path | Goroutine dies silently on panic unless recover+log |
|
||||
| Context canceled but operation continues | Ignored `ctx.Err()` check |
|
||||
| `json.Unmarshal` of zero-value struct field | Input missing the key; silently zero |
|
||||
| Closed channel read returning zero value | Consumer doesn't check `ok`; reads forever |
|
||||
|
||||
---
|
||||
|
||||
## Phase 9 cleanup specifics
|
||||
|
||||
```bash
|
||||
# Kill dlv sessions
|
||||
pkill -f 'dlv' || true
|
||||
lsof -iTCP:2345 -sTCP:LISTEN -nP 2>/dev/null # dlv default
|
||||
|
||||
# Kill pprof HTTP endpoint if you started it just for this session
|
||||
lsof -iTCP:6060 -sTCP:LISTEN -nP 2>/dev/null
|
||||
|
||||
# Revert any `fmt.Println("DEBUG: ...")` or `log.Printf("DEBUG: ...")` additions
|
||||
git diff | grep -E '(fmt\.Println\("DEBUG|log\.Printf\("DEBUG|println!)'
|
||||
git checkout <file>
|
||||
|
||||
# Unset env vars
|
||||
unset GODEBUG
|
||||
```
|
||||
@@ -0,0 +1,484 @@
|
||||
# Native Binary Debugging (No Source / Reverse Engineering)
|
||||
|
||||
For binaries where you don't have trustworthy source: stripped production builds, third-party closed libs, malware, CTF challenges, firmware, vendored libs whose docs lie. The workflow is specific; doing it out of order wastes days.
|
||||
|
||||
This reference **coordinates** the triage and dynamic work. The heavy tools each have their own reference:
|
||||
- **Static decompilation** → [tools/ghidra.md](../tools/ghidra.md)
|
||||
- **Interactive debugging** → [tools/pwndbg.md](../tools/pwndbg.md)
|
||||
- **Scripted interaction / exploitation** → [tools/pwntools.md](../tools/pwntools.md)
|
||||
|
||||
Read those before using them — especially Ghidra, which has a surprising amount of workflow that's not obvious.
|
||||
|
||||
---
|
||||
|
||||
## ⚠️ STOP — is this actually a stripped C/C++ binary?
|
||||
|
||||
A growing share of "binaries" are actually **bundled high-level apps** — Bun SEA, Node SEA, Deno compile, pkg, nexe, Electron, Tauri, PyInstaller. Their workflow is completely different: the high-level source is recoverable with the right per-bundler tool (often plaintext, sometimes V8 cache / `.pyc` / eszip needing extra tooling), and Ghidra against the runtime VM wastes hours.
|
||||
|
||||
Quick check:
|
||||
|
||||
```bash
|
||||
file ./target # Mach-O / ELF / PE - inconclusive
|
||||
du -h ./target # 50 MB+ for a "simple CLI" → suspect bundled
|
||||
strings -n 12 ./target | rg -iE 'bun|node_modules|webpack|esbuild|deno|pkg/lib|electron|pyinstaller|nexe|NODE_SEA_FUSE|tauri' | head -5
|
||||
```
|
||||
|
||||
**If any hits** → close this file, open [bundled-js-binary.md](bundled-js-binary.md) instead. Following the Ghidra/pwndbg path on a bundled-app binary wastes hours decompiling the runtime VM while the app-level bundle is recoverable with the right per-bundler tool (plaintext for Bun/pkg/nexe/Electron-asar; eszip / V8-cache / `.pyc` for Deno / Node SEA / PyInstaller).
|
||||
|
||||
If `file` says "Mach-O" or "ELF", `du` is < 20 MB, and the strings check is empty → continue here.
|
||||
|
||||
---
|
||||
|
||||
## The workflow (do these in order)
|
||||
|
||||
Every step's output is input to the next. Skipping steps means guessing later.
|
||||
|
||||
```
|
||||
[1] Triage → what kind of binary is this?
|
||||
[2] Dynamic tracing → what syscalls / libcalls does it make?
|
||||
[3] Static analysis → what does it DO, in readable form? (Ghidra)
|
||||
[4] Dynamic debug → confirm hypotheses at runtime (pwndbg)
|
||||
[5] Scripted repro → lock the bug with a pwntools script
|
||||
[6] TDD + fix / report
|
||||
```
|
||||
|
||||
Steps 1 and 2 are fast (minutes). Step 3 is slow (tens of minutes to hours depending on size). Don't skip 1-2 and go straight to Ghidra — the triage output tells you what to focus on inside Ghidra.
|
||||
|
||||
---
|
||||
|
||||
## [1] Triage — 5-minute fingerprint
|
||||
|
||||
```bash
|
||||
# Basic identity
|
||||
file ./target
|
||||
# elf, mach-o, pe? 32/64-bit? dynamically linked? stripped?
|
||||
|
||||
# Architecture details
|
||||
readelf -h ./target # ELF header: entry point, arch, type
|
||||
lipo -info ./target 2>/dev/null # macOS: universal binary?
|
||||
|
||||
# Interesting strings (often leaks function names, error messages, URLs, API keys)
|
||||
strings -n 8 ./target | head -100
|
||||
strings -n 8 ./target | grep -iE '(http|/api/|error|debug|version)'
|
||||
|
||||
# Imported symbols (what does it link against?)
|
||||
nm -D ./target 2>/dev/null # dynamic symbols
|
||||
objdump -T ./target 2>/dev/null # same, alternate tool
|
||||
readelf -d ./target # dynamic section (NEEDED libs)
|
||||
ldd ./target 2>/dev/null # resolved library paths
|
||||
|
||||
# Security posture (affects what exploits / bugs are possible)
|
||||
checksec --file=./target # requires pwntools or installing checksec
|
||||
# NX, PIE, RELRO, stack canary, FORTIFY
|
||||
|
||||
# Is it stripped?
|
||||
nm ./target 2>/dev/null | head # empty? stripped. full? not stripped.
|
||||
file ./target # will say "stripped" or "not stripped"
|
||||
```
|
||||
|
||||
### ⚠️ `strings -n N` silently drops short content
|
||||
|
||||
`strings` prints runs of printable characters of length **≥ N**. With `-n 8`, **anything shorter than 8 chars sandwiched between non-printable bytes is dropped silently**. This includes:
|
||||
|
||||
- Short identifier interpolations in templates (`${x}`, `${i}`, `${R}`)
|
||||
- Short embedded constants (`v3`, `null`, integer immediates as bytes)
|
||||
- Short error codes between binary padding
|
||||
|
||||
Real example: a JavaScript template literal `<INSTRUCTIONS>\n${x}\n</INSTRUCTIONS>` came out of `strings -n 8` as `<INSTRUCTIONS>\n</INSTRUCTIONS>` — the `${x}` (4 chars) was dropped. A consumer reading the dump would conclude the template was empty. It is not.
|
||||
|
||||
**Use `strings` only for fingerprinting (Phase 1).** For any extraction whose correctness matters, **read bytes directly**:
|
||||
|
||||
```bash
|
||||
# Count occurrences of a needle
|
||||
LC_ALL=C grep -aoc 'NEEDLE' ./target
|
||||
|
||||
# Find offsets
|
||||
LC_ALL=C grep -aob 'NEEDLE' ./target | head
|
||||
|
||||
# Or via Python for byte-precise context
|
||||
python3 -c "
|
||||
import sys
|
||||
data = open('./target','rb').read()
|
||||
needle = b'NEEDLE'
|
||||
pos = data.find(needle)
|
||||
print(repr(data[max(0,pos-100):pos+200]))
|
||||
"
|
||||
```
|
||||
|
||||
If you must keep using `strings`, lower the threshold: `strings -n 1 -t x ./target | rg ...`. The signal-to-noise drops sharply but short content is preserved.
|
||||
|
||||
Write the triage summary to the journal:
|
||||
|
||||
```markdown
|
||||
## Binary triage
|
||||
- Type: <ELF 64-bit, dynamically linked, stripped>
|
||||
- Arch: <x86_64 | arm64 | ...>
|
||||
- Libs: <libc, openssl, libcurl>
|
||||
- Security: <NX, PIE, Partial RELRO, no canary>
|
||||
- Interesting strings: <short list>
|
||||
- First hypothesis surface: <which function / area looks most relevant>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## [2] Dynamic tracing — what does it actually call?
|
||||
|
||||
These are cheap — run them before Ghidra to orient yourself.
|
||||
|
||||
### Linux: strace + ltrace
|
||||
|
||||
```bash
|
||||
# System calls
|
||||
strace -f -o trace.out ./target arg1 arg2
|
||||
strace -f -e trace=network ./target # filter to network syscalls
|
||||
strace -f -e trace=file ./target # filter to file ops
|
||||
|
||||
# Library calls (less useful when stripped but still informative)
|
||||
ltrace -f -o ltrace.out ./target
|
||||
ltrace -f -e 'str*+mem*' ./target # filter to string/mem functions
|
||||
```
|
||||
|
||||
### macOS: Mach-O specifics
|
||||
|
||||
**SIP block reality check.** With System Integrity Protection enabled (default on every modern macOS), `dtruss` / `dtrace` will **silently fail** to attach to:
|
||||
- Anything in `/usr`, `/bin`, `/sbin`, `/System`
|
||||
- Apple-signed binaries (Xcode CLT, Homebrew formulae from Apple-distributed taps)
|
||||
- Notarized vendor binaries (Bun, Deno, Docker Desktop, etc.)
|
||||
|
||||
`dtruss ./target` will appear to run but produce zero events. This is not a bug; it is the SIP design. Disabling SIP requires a Recovery Mode reboot — usually not worth it. Use the alternatives below.
|
||||
|
||||
```bash
|
||||
# dtruss — works only when SIP allows it (your own unsigned binaries)
|
||||
sudo dtruss -f ./target 2>&1 | head -20 # equivalent to strace
|
||||
# If output is suspiciously empty → SIP blocked it. Switch to lldb or app-level logging.
|
||||
```
|
||||
|
||||
**Mach-O metadata inspection (no SIP issues, no debugger needed):**
|
||||
|
||||
```bash
|
||||
# Architecture and slices
|
||||
file ./target # arm64 / x86_64 / universal
|
||||
lipo -info ./target # which architectures included
|
||||
lipo -thin arm64 ./target -output ./target-arm64 # extract one slice for analysis
|
||||
|
||||
# Headers & load commands (segments, dylibs, code-signature pointer)
|
||||
otool -h ./target # Mach header (cputype, ncmds, flags)
|
||||
otool -l ./target | head -100 # load commands; entitlements live in code-signature blob, see codesign below
|
||||
|
||||
# Dynamic library dependencies (macOS equivalent of ldd)
|
||||
otool -L ./target # linked dylibs with versions
|
||||
dyld_info ./target # macOS 13+, more detailed than otool -L
|
||||
|
||||
# Disassembly
|
||||
otool -tv ./target | head -200 # quick disassembly without Ghidra
|
||||
otool -tV ./target # with symbol-resolved branches
|
||||
|
||||
# Imported / exported symbols (Apple `nm`, NOT GNU)
|
||||
nm -u ./target # undefined references = imports
|
||||
nm -gU ./target # external defined = exports
|
||||
# Note: GNU `-D`/dynamic flags are not honored on Apple `nm`; use the above forms.
|
||||
symbols -fullSourcePath -onlyWithDebugInfo ./target # if any debug info survives
|
||||
|
||||
# Code signature & entitlements (entitlements come from codesign, NOT otool)
|
||||
codesign -dv --entitlements :- ./target 2>&1 # signature info + entitlements XML on stdout
|
||||
spctl --assess --type execute -vv ./target # Gatekeeper assessment
|
||||
|
||||
# Cert chain — extract to a temp dir to avoid creating files named -0/-1 in cwd
|
||||
tmp=$(mktemp -d)
|
||||
codesign -dvv --extract-certificates="$tmp/cert" ./target 2>&1
|
||||
ls -la "$tmp"
|
||||
# rm -rf "$tmp" # journal first, clean up later
|
||||
|
||||
# Strings inside specific segments only (less noise than full-binary strings)
|
||||
otool -s __TEXT __cstring ./target # C string section
|
||||
otool -s __TEXT __const ./target # constants section
|
||||
```
|
||||
|
||||
**Interactive debugging on macOS — use `lldb`, not `gdb`.**
|
||||
|
||||
GDB on macOS requires a self-signed code-signing certificate (`codesign --entitlements gdb.entitlements --sign gdb-cert /opt/homebrew/bin/gdb`) and even then is unreliable on arm64. **Use `lldb` directly** — it ships with Xcode CLT and works without configuration.
|
||||
|
||||
```bash
|
||||
# Start lldb
|
||||
lldb ./target
|
||||
|
||||
# Set arguments
|
||||
(lldb) settings set target.run-args arg1 arg2
|
||||
|
||||
# Run with breakpoints
|
||||
(lldb) breakpoint set --name function_name # symbol-based
|
||||
(lldb) breakpoint set --address 0x1000034c0 # address-based
|
||||
(lldb) breakpoint set --regex '.*decode.*' # regex over symbols
|
||||
|
||||
# Run / step / inspect
|
||||
(lldb) run
|
||||
(lldb) bt # backtrace
|
||||
(lldb) frame variable # locals
|
||||
(lldb) register read # all registers
|
||||
(lldb) memory read --size 8 --format x --count 16 $sp # 16 qwords from stack
|
||||
(lldb) disassemble --frame # current function
|
||||
(lldb) image list # loaded modules
|
||||
(lldb) image lookup -a 0x1000034c0 # which module + symbol owns this address
|
||||
|
||||
# Process attach to running process
|
||||
(lldb) process attach --pid 12345
|
||||
(lldb) process attach --name target # attach by name
|
||||
|
||||
# Print Mach-O specific
|
||||
(lldb) image dump sections ./target
|
||||
(lldb) image dump symtab ./target
|
||||
```
|
||||
|
||||
**Function interception via `DYLD_INSERT_LIBRARIES`** (macOS equivalent of `LD_PRELOAD`):
|
||||
|
||||
```bash
|
||||
# Build a shim dylib that overrides specific functions
|
||||
# Then run target with it preloaded
|
||||
DYLD_INSERT_LIBRARIES=./shim.dylib DYLD_FORCE_FLAT_NAMESPACE=1 ./target
|
||||
```
|
||||
|
||||
DYLD_INSERT works in the unrestricted case but is blocked in three distinct scenarios — distinguish them when diagnosing why your shim didn't load:
|
||||
|
||||
1. **SIP / restricted process** (target has the `__RESTRICT,__restrict` section, is setuid/setgid, or is a platform/Apple-signed binary): dyld unconditionally strips all `DYLD_*` env vars before the process starts. Nothing you set will reach the target.
|
||||
2. **Hardened runtime + library validation** (`CS_RUNTIME` flag set, `com.apple.security.cs.disable-library-validation` entitlement absent): the process accepts `DYLD_INSERT_LIBRARIES` but **rejects** loading any dylib that isn't signed by the same Team ID or by Apple. Symptom: shim is found but not loaded; check `log show --predicate 'eventMessage CONTAINS "library validation failed"'`.
|
||||
3. **Notarization / Gatekeeper translocation**: the binary may be running from a translocated path; relative paths in `DYLD_INSERT_LIBRARIES` won't resolve. Use absolute paths.
|
||||
|
||||
Check each:
|
||||
|
||||
```bash
|
||||
# Restrict segment present? (case 1)
|
||||
otool -l ./target | grep -A2 __RESTRICT
|
||||
# Hardened runtime flag? (case 2)
|
||||
codesign -d --verbose=4 ./target 2>&1 | grep -iE 'flags=|CodeDirectory'
|
||||
# Look for "0x10000(runtime)" or similar in the flags line.
|
||||
# Disable-library-validation entitlement?
|
||||
codesign -d --entitlements :- ./target 2>&1 | grep disable-library-validation
|
||||
```
|
||||
|
||||
**App-level debug logging (always works, ignores SIP):**
|
||||
|
||||
When debugger attach is blocked, fall back to maximizing the app's own logging:
|
||||
|
||||
```bash
|
||||
# Try common patterns
|
||||
APP_DEBUG=1 APP_LOG_LEVEL=debug APP_LOG_FILE=/tmp/trace.log ./target
|
||||
NSDebugEnabled=YES ./target # Cocoa apps
|
||||
OS_ACTIVITY_MODE=debug ./target # os_log subsystem
|
||||
|
||||
# Then read os_log unified logging stream live
|
||||
log stream --predicate 'process == "target"' --level debug
|
||||
|
||||
# Or extract historical logs
|
||||
log show --predicate 'process == "target"' --last 1h --info --debug
|
||||
```
|
||||
|
||||
This is the **partial-runtime-evidence path** for macOS. See [methodology/partial-runtime-evidence.md](../methodology/partial-runtime-evidence.md) for how to combine app-level logs with static analysis when wire-level capture is blocked.
|
||||
|
||||
**Network capture on macOS (TLS-decrypted):**
|
||||
|
||||
```bash
|
||||
# 1. Find the active network service (don't assume "Wi-Fi"):
|
||||
# Map the default-route interface to the matching networksetup service name.
|
||||
networksetup -listallnetworkservices # show options
|
||||
DEFAULT_IF=$(route -n get default 2>/dev/null | awk '/interface:/ {print $2}')
|
||||
echo "Default-route interface: $DEFAULT_IF"
|
||||
# Match the interface (en0, en1, ...) back to a service name:
|
||||
SERVICE=$(networksetup -listallhardwareports | awk -v iface="$DEFAULT_IF" '
|
||||
/^Hardware Port:/ { hp = substr($0, index($0,$3)) }
|
||||
/^Device:/ { if ($2 == iface) print hp }
|
||||
')
|
||||
if [ -z "$SERVICE" ]; then
|
||||
echo "Could not auto-detect active service. Pick one from -listallnetworkservices manually." >&2
|
||||
echo "Aborting proxy setup." >&2
|
||||
false # signal failure but stay safe at top level
|
||||
else
|
||||
echo "Using service: $SERVICE"
|
||||
fi
|
||||
|
||||
# 2. JOURNAL the original proxy state before changing it (REQUIRED for safe rollback):
|
||||
networksetup -getwebproxy "$SERVICE" # save this output to journal
|
||||
networksetup -getsecurewebproxy "$SERVICE" # save this too
|
||||
|
||||
# 3. Start mitmproxy with persistent CA at ~/.mitmproxy/
|
||||
mitmproxy --listen-host 127.0.0.1 --listen-port 8888 &
|
||||
|
||||
# 4. Trust the mitmproxy CA system-wide if the target uses URLSession or any framework
|
||||
# that ignores HTTPS_PROXY/SSL_CERT_FILE (most macOS-native apps do):
|
||||
sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ~/.mitmproxy/mitmproxy-ca-cert.pem
|
||||
|
||||
# 5. Two routing options. Try env-var first; fall back to system proxy:
|
||||
# 5a. Apps that honor env vars (most CLIs):
|
||||
HTTPS_PROXY=http://127.0.0.1:8888 SSL_CERT_FILE=~/.mitmproxy/mitmproxy-ca-cert.pem ./target ...
|
||||
|
||||
# 5b. Apps that use URLSession / system network config (most GUI apps, Bun, some CLIs):
|
||||
networksetup -setwebproxy "$SERVICE" 127.0.0.1 8888
|
||||
networksetup -setsecurewebproxy "$SERVICE" 127.0.0.1 8888
|
||||
|
||||
# 6. Cleanup — RESTORE original state from journal, untrust CA:
|
||||
networksetup -setwebproxystate "$SERVICE" off
|
||||
networksetup -setsecurewebproxystate "$SERVICE" off
|
||||
sudo security delete-certificate -c "mitmproxy" /Library/Keychains/System.keychain
|
||||
```
|
||||
|
||||
**Critical**: forgetting step 6 leaves all your subsequent traffic mis-routed and silently MITM-able. Journal every step.
|
||||
|
||||
### What to look for
|
||||
|
||||
| Observation | Hypothesis |
|
||||
|---|---|
|
||||
| `open("/etc/secret-config", ...)` | Reads unexpected config; look at what it does with contents |
|
||||
| `connect(... 1.2.3.4:443)` | Phones home or depends on an external service |
|
||||
| `getenv("FOO")` returning NULL | Env var expected but not set |
|
||||
| Repeated `poll`/`epoll_wait` with no progress | Stuck on I/O; check downstream |
|
||||
| `SIGSEGV` caught by signal handler | Custom crash recovery — often hides the real bug |
|
||||
| `dlopen("libfoo.so.42")` | Dynamic plugin loading; check plugin path |
|
||||
|
||||
---
|
||||
|
||||
## [3] Static analysis with Ghidra
|
||||
|
||||
When triage + tracing have narrowed you to "something in function X" or "the crypto routine is weird", open Ghidra.
|
||||
|
||||
**Open [tools/ghidra.md](../tools/ghidra.md) before launching Ghidra** — the import / analyze / decompile workflow is not obvious and first-time users waste an hour figuring it out.
|
||||
|
||||
Ghidra's decompiler turns machine code into readable-ish C. That's usually what you want. Stay in the Decompiler view; drop to Listing (disassembly) only when the decompiler punts.
|
||||
|
||||
---
|
||||
|
||||
## [4] Dynamic debugging with pwndbg
|
||||
|
||||
Once static analysis gives you a hypothesis ("this branch at 0x401234 is where the validation fails"), confirm it at runtime with pwndbg.
|
||||
|
||||
**Open [tools/pwndbg.md](../tools/pwndbg.md) before launching gdb.** Pwndbg gives you the context view (registers / stack / disasm / code all visible at once) which is essential for binary debugging.
|
||||
|
||||
Typical pwndbg flow:
|
||||
|
||||
```
|
||||
$ gdb ./target # pwndbg loads automatically if installed
|
||||
pwndbg> break *0x401234 # break at the address static analysis flagged
|
||||
pwndbg> run arg1 arg2
|
||||
# At the breakpoint:
|
||||
pwndbg> context # registers + stack + disasm
|
||||
pwndbg> telescope $rdi # walk pointers at $rdi
|
||||
pwndbg> x/20xw $rsp # raw dump of stack
|
||||
pwndbg> ni / si # step next / step instruction
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## [5] Scripted reproduction with pwntools
|
||||
|
||||
Once you have a hypothesis with a concrete repro input, lock it down with pwntools. This is the "failing test" equivalent for binaries.
|
||||
|
||||
**Open [tools/pwntools.md](../tools/pwntools.md)** — the Process/Remote/ELF/context APIs are the foundation.
|
||||
|
||||
```python
|
||||
from pwn import *
|
||||
|
||||
context.binary = elf = ELF('./target')
|
||||
|
||||
p = process('./target')
|
||||
p.sendlineafter(b'> ', b'<trigger input that reproduces the bug>')
|
||||
result = p.recvall(timeout=3)
|
||||
assert b'expected-output-when-fixed' in result, f'bug repro: {result}'
|
||||
```
|
||||
|
||||
This script is now your "red test". When the fix is applied, the script should pass (or the assertion should be inverted for negative tests — e.g. "the crash string should NOT appear").
|
||||
|
||||
---
|
||||
|
||||
## [6] Fixing a binary bug you can't recompile
|
||||
|
||||
Three options, in preference order:
|
||||
|
||||
### Option A: Patch at the source (if you have it)
|
||||
|
||||
If the bug is in your own code and source is available, fix it there and rebuild. Standard TDD path.
|
||||
|
||||
### Option B: Binary patch
|
||||
|
||||
For tiny fixes (one byte, one branch inversion):
|
||||
|
||||
```bash
|
||||
# Identify the exact byte offset
|
||||
# e.g. Ghidra says the bug is at 0x401234 = file offset 0x1234
|
||||
printf '\x90\x90' | dd of=./target bs=1 seek=$((0x1234)) conv=notrunc
|
||||
```
|
||||
|
||||
Journal the exact `dd` command and the original bytes so you can revert.
|
||||
|
||||
### Option C: Wrap / shim
|
||||
|
||||
If you can't patch the binary, write a shim library (LD_PRELOAD on Linux, DYLD_INSERT_LIBRARIES on macOS) that overrides the buggy function. pwntools has examples.
|
||||
|
||||
### Option D: Report upstream
|
||||
|
||||
If it's a third-party binary and none of the above are feasible, the "fix" is a high-quality bug report with:
|
||||
- Full triage summary
|
||||
- Reproducible pwntools script
|
||||
- Ghidra decompilation of the buggy function
|
||||
- Hypothesis about the root cause
|
||||
- Recommended patch sketch (in C or pseudocode)
|
||||
|
||||
---
|
||||
|
||||
## Silent-failure patterns in native binaries
|
||||
|
||||
| Pattern | Why it's silent |
|
||||
|---|---|
|
||||
| Ignored libc return codes (`read`, `write`, `malloc`) | Bug continues with garbage data; no check |
|
||||
| Signal handler swallows SIGSEGV | Crash converted to "something didn't work"; no log |
|
||||
| `setjmp`/`longjmp` unwinding over cleanup | Resources leak silently |
|
||||
| Thread-local error state never read (`errno`, `GetLastError`) | Error happened, nobody asked |
|
||||
| Recovered assertion failure in release build | `assert` compiled out; precondition violations silently corrupt |
|
||||
| Dangling pointer reads after free | Often looks like valid data until it doesn't |
|
||||
|
||||
---
|
||||
|
||||
## Phase 9 cleanup specifics
|
||||
|
||||
```bash
|
||||
# Kill debugger sessions
|
||||
pkill -f 'gdb' || true
|
||||
pkill -f 'lldb' || true
|
||||
|
||||
# Ghidra scratch projects (if made just for this session)
|
||||
# Named something like ~/ghidra-projects/debug-<timestamp>:
|
||||
ls -la ~/ghidra-projects/ 2>/dev/null
|
||||
# rm -rf ~/ghidra-projects/debug-scratch # only if the journal says to
|
||||
|
||||
# Core dumps left from crashes
|
||||
rm -f ./core ./core.* ~/core.*
|
||||
|
||||
# strace/ltrace output files
|
||||
rm -f trace.out ltrace.out
|
||||
|
||||
# If you made a binary patch (Option B above), confirm revert
|
||||
# The journal should have the original bytes — restore them:
|
||||
# printf '<original-bytes>' | dd of=./target bs=1 seek=<offset> conv=notrunc
|
||||
|
||||
# Trace-output files
|
||||
rm -f /tmp/debug-*.bin /tmp/debug-*.strace /tmp/debug-*.ltrace
|
||||
|
||||
# macOS-specific:
|
||||
# Restore proxy settings if you set them (CRITICAL — leaves system traffic mis-routed otherwise)
|
||||
# Use the SAME $SERVICE you used when enabling the proxy (read it from the journal).
|
||||
# Do NOT hardcode "Wi-Fi" — many machines route traffic over Ethernet, USB tether, or a VPN service.
|
||||
[ -n "$SERVICE" ] && {
|
||||
networksetup -setwebproxystate "$SERVICE" off 2>/dev/null
|
||||
networksetup -setsecurewebproxystate "$SERVICE" off 2>/dev/null
|
||||
}
|
||||
# Or restore explicitly from the journaled original state — see the proxy section above.
|
||||
|
||||
# Stop mitmproxy
|
||||
pkill -f 'mitmproxy' 2>/dev/null
|
||||
|
||||
# Remove DYLD shim libraries you built
|
||||
rm -f /tmp/*-shim.dylib
|
||||
|
||||
# Clear extracted strings dumps (these can be huge and may contain secrets)
|
||||
rm -f /tmp/*-strings*.txt
|
||||
|
||||
# Verify hostname resolution returns to normal (mitmproxy can leave entries)
|
||||
scutil --dns | head -20
|
||||
```
|
||||
@@ -0,0 +1,260 @@
|
||||
# Node.js / tsx / ts-node / Bun / Deno Debugging
|
||||
|
||||
Covers Node 18+, tsx, ts-node, Bun, Deno. Launch recipes, inspector protocol usage, the `node inspect` CLI, and the **tsx source-map silent-failure** that costs people days.
|
||||
|
||||
---
|
||||
|
||||
## Environment detection (Phase 0)
|
||||
|
||||
```bash
|
||||
node --version
|
||||
cat package.json | head -40
|
||||
|
||||
# Which JS runtime launches the app? (order them; the first match wins)
|
||||
ls node_modules/.bin/tsx 2>/dev/null && echo 'has tsx'
|
||||
ls node_modules/.bin/ts-node 2>/dev/null && echo 'has ts-node'
|
||||
ls node_modules/.bin/vitest 2>/dev/null && echo 'has vitest'
|
||||
which bun 2>/dev/null && bun --version
|
||||
which deno 2>/dev/null && deno --version
|
||||
|
||||
# Source-map situation
|
||||
grep -E '"sourceMap"|"inlineSources"' tsconfig.json 2>/dev/null
|
||||
grep -l '//# sourceMappingURL' dist/*.js 2>/dev/null | head -3
|
||||
|
||||
# Debug-relevant ports
|
||||
lsof -iTCP:9229 -sTCP:LISTEN -nP 2>/dev/null
|
||||
lsof -iTCP:9230 -sTCP:LISTEN -nP 2>/dev/null
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🚨 The tsx + `node inspect` CLI silent-failure (READ THIS)
|
||||
|
||||
`tsx` transpiles each `.ts` file on the fly and emits an inline source map. V8 Inspector registers the module with its `.ts` path (so it shows up in the debugger's `scripts` list), **but the `node inspect` CLI REPL does not resolve source-map line numbers reliably**. Setting `sb('session.ts', 285)` will show a "pending" breakpoint that **never fires even after the module loads**.
|
||||
|
||||
The breakpoint list will happily display it, so you think it's set. It isn't.
|
||||
|
||||
### Three reliable workarounds
|
||||
|
||||
| Workaround | When to use | Downside |
|
||||
|---|---|---|
|
||||
| **`debugger;` statement in source** | You can edit the source, CLI required | Requires source edit + revert |
|
||||
| **Chrome DevTools GUI** (`chrome://inspect`) | CLI not required, faster iteration | Not usable if user specifically asked for CLI |
|
||||
| **Debug the built `dist/` JS** | Source maps are working end-to-end | Requires `npm run build` on every source change |
|
||||
|
||||
The `debugger;` statement is the most reliable. Journal the edit — revert at Phase 9.
|
||||
|
||||
---
|
||||
|
||||
## Launch recipes by runtime
|
||||
|
||||
### Node (plain JS / compiled TS)
|
||||
|
||||
```bash
|
||||
# Break on first line, wait for debugger to attach
|
||||
node --inspect-brk=9229 dist/index.js
|
||||
|
||||
# Attach immediately, don't block startup — pair with debugger; statements
|
||||
node --inspect=9229 dist/index.js
|
||||
|
||||
# Wait for debugger to attach, THEN run (new in Node 20.15+)
|
||||
node --inspect-wait=9229 dist/index.js
|
||||
|
||||
# Source maps in stack traces (always a good idea in debug builds)
|
||||
node --enable-source-maps --inspect dist/index.js
|
||||
```
|
||||
|
||||
### tsx
|
||||
|
||||
```bash
|
||||
# The tsx runner is --import-compatible, so these work:
|
||||
node --inspect-brk=9229 --import tsx index.ts
|
||||
node --inspect=9229 --import tsx index.ts
|
||||
|
||||
# If user prefers invoking tsx directly, this also works but is less explicit:
|
||||
NODE_OPTIONS='--inspect-brk=9229' npx tsx index.ts
|
||||
|
||||
# ⚠️ tsx watch + inspector = inspector reloads per file change
|
||||
# Debug without watch:
|
||||
node --inspect=9229 --import tsx index.ts # (no `watch`)
|
||||
```
|
||||
|
||||
### ts-node (legacy but still encountered)
|
||||
|
||||
```bash
|
||||
node --inspect-brk -r ts-node/register src/index.ts
|
||||
# ESM (ts-node's ESM loader is fragile — if possible, migrate to tsx):
|
||||
node --inspect --loader ts-node/esm src/index.ts
|
||||
```
|
||||
|
||||
### Bun (WebKit Inspector Protocol, NOT V8)
|
||||
|
||||
```bash
|
||||
bun --inspect src/index.ts # opens debug.bun.sh URL
|
||||
bun --inspect-brk src/index.ts # break on start
|
||||
bun --inspect-wait src/index.ts # wait for attach
|
||||
bun test --inspect-brk # debug test runner
|
||||
```
|
||||
|
||||
**Critical**: Bun uses WebKit Inspector Protocol, not V8. `chrome://inspect` cannot connect directly. Use `debug.bun.sh` or the (currently buggy, per Bun docs) VS Code extension.
|
||||
|
||||
### Deno (native V8, Chrome DevTools / VS Code compatible)
|
||||
|
||||
```bash
|
||||
deno run --inspect-brk --allow-all src/main.ts
|
||||
deno test --inspect-brk --filter "auth"
|
||||
```
|
||||
|
||||
Deno is the smoothest TS debugging experience — native V8 inspector, no source-map workarounds.
|
||||
|
||||
### Vitest
|
||||
|
||||
```bash
|
||||
# Single worker required — inspector can't attach to multiple workers
|
||||
vitest --inspect-brk --no-file-parallelism
|
||||
vitest --inspect-brk --browser --no-file-parallelism # browser mode
|
||||
```
|
||||
|
||||
Without `--no-file-parallelism`, breakpoints won't fire because the process Vitest spawns workers in isn't the one listening on the inspector port.
|
||||
|
||||
---
|
||||
|
||||
## Attaching with `node inspect` CLI
|
||||
|
||||
```bash
|
||||
node inspect 127.0.0.1:9229 # attach to an existing --inspect process
|
||||
```
|
||||
|
||||
Core commands at the `debug>` prompt:
|
||||
|
||||
```
|
||||
cont, c resume until next break / debugger;
|
||||
next, n step over
|
||||
step, s step into
|
||||
out, o step out
|
||||
pause pause a running process
|
||||
bt backtrace
|
||||
scripts list all modules V8 has loaded (incl. tsx-transpiled .ts)
|
||||
sb(N) set breakpoint at line N of current file
|
||||
sb('file', N) set breakpoint at line N of matching file (⚠️ unreliable with tsx)
|
||||
sb(func) set breakpoint at function reference
|
||||
cb(N), cb('file', N) clear breakpoint
|
||||
breakpoints list breakpoints (shows pending ones, doesn't tell you they'll never fire)
|
||||
watch('expr') persistent watch expression
|
||||
watchers show watchers
|
||||
exec('expr') evaluate expression in paused frame's scope
|
||||
repl drop into full REPL with frame's scope
|
||||
restart restart the debuggee
|
||||
kill kill the debuggee
|
||||
```
|
||||
|
||||
**`exec('expr')` is the most powerful tool in this CLI** — it evaluates any JS in the paused frame and returns the value. Use it heavily.
|
||||
|
||||
---
|
||||
|
||||
## `exec()` patterns that resolve hypotheses fast
|
||||
|
||||
At a breakpoint, these queries resolve most LLM / agent / async bugs in one line each:
|
||||
|
||||
```js
|
||||
// Agent / LLM state
|
||||
exec('this.agent.state.messages.length')
|
||||
exec('this.agent.state.messages.map(m => m.role)')
|
||||
exec('JSON.stringify(this.agent.state.messages.at(-1)).substring(0, 500)')
|
||||
exec('this.agent.state.messages.at(-1).errorMessage') // silent-error sentinel
|
||||
exec('this.agent.state.messages.at(-1).stopReason')
|
||||
exec('JSON.stringify(this.agent.state.usage)') // undefined / all-zero = failed call
|
||||
exec('this.agent.state.model.baseUrl') // catch hardcoded vs env-var
|
||||
|
||||
// Env / config at runtime
|
||||
exec('process.env.RELEVANT_VAR')
|
||||
exec('Object.keys(process.env).filter(k => k.startsWith("ANTHROPIC"))')
|
||||
exec('this.config')
|
||||
|
||||
// Async / timing
|
||||
exec('Date.now() - this._turnStartedAt')
|
||||
exec('this._activePromises?.size')
|
||||
|
||||
// HTTP request/response in-flight
|
||||
exec('JSON.stringify(req.body).length')
|
||||
exec('res.statusCode')
|
||||
exec('res.headersSent')
|
||||
|
||||
// What's actually running
|
||||
exec('process.version')
|
||||
exec('process.cwd()')
|
||||
exec('process.argv')
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Silent-failure patterns in Node
|
||||
|
||||
These are the patterns that most commonly look like success but aren't. Always check when a response is "too fast" or "too empty":
|
||||
|
||||
| Signal | What it means |
|
||||
|---|---|
|
||||
| HTTP 200 + `content: ""` | Silent error swallowed |
|
||||
| HTTP 200 + response in <1s for an LLM call | Too fast for a real Claude/GPT call; something short-circuited |
|
||||
| `usage: { totalTokens: 0 }` | LLM SDK returned a stub without making the call |
|
||||
| `stopReason: "error" + content: []` | SDK packaged an error into a "success" message |
|
||||
| Unhandled promise rejection with no log | Caller forgot to `await`, or `.catch(() => {})` |
|
||||
| `try { await x(); } catch {}` | Error eaten, no log |
|
||||
| `void somePromise()` | Explicit opt-out of error propagation; often a bug |
|
||||
| Callback-style API where callback never fires | Error happened before callback scheduled |
|
||||
| Handler returns `res.json(...)` twice | Second call is silent on some Express versions |
|
||||
|
||||
When you find one, add a temporary `console.error('[DEBUG]', ...)` to make it loud — journal it, revert at Phase 9.
|
||||
|
||||
---
|
||||
|
||||
## tmux session layout (two sessions, one purpose each)
|
||||
|
||||
```bash
|
||||
# Long-running inspected process
|
||||
tmux new-session -d -s debug-server -c "$PWD"
|
||||
tmux send-keys -t debug-server 'node --inspect=9229 --import tsx index.ts' Enter
|
||||
|
||||
# Interactive debugger client (separate pane for readability)
|
||||
tmux new-session -d -s debug-client -c "$PWD"
|
||||
tmux send-keys -t debug-client 'node inspect 127.0.0.1:9229' Enter
|
||||
|
||||
# Non-blocking pane inspection from the outside
|
||||
tmux capture-pane -p -t debug-server -S -50
|
||||
```
|
||||
|
||||
Journal both session names. Kill both at Phase 9:
|
||||
|
||||
```bash
|
||||
tmux kill-session -t debug-server
|
||||
tmux kill-session -t debug-client
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## When to abandon the CLI and switch to Chrome DevTools
|
||||
|
||||
The user's preference for CLI is valid and should be respected. But you may recommend a switch in one short sentence if ANY of these hold:
|
||||
|
||||
- You hit source-map resolution failures (`sb('file', line)` not firing) AND the fix is time-sensitive
|
||||
- You need to watch many values simultaneously (GUI watch panel is faster to scan)
|
||||
- You're stepping through async-heavy code where CLI step semantics get murky across microtask boundaries
|
||||
|
||||
Phrase as a note, not a request: "I can push through with `debugger;` statements in CLI. If we hit three or more of these in a row, switching to `chrome://inspect` GUI would cut cycle time in half — your call."
|
||||
|
||||
---
|
||||
|
||||
## Phase 9 cleanup specifics
|
||||
|
||||
```bash
|
||||
# Revert source-level debug statements
|
||||
git diff | grep -E '(debugger;|console\.log\(.*DEBUG|\[ARBITER-DEBUG|\[DEBUG)'
|
||||
# Revert any matching files:
|
||||
git checkout <file>
|
||||
|
||||
# Kill inspector-attached processes
|
||||
pkill -f 'node --inspect' || true
|
||||
pkill -f 'bun --inspect' || true
|
||||
pkill -f 'deno.*--inspect' || true
|
||||
lsof -iTCP:9229 -sTCP:LISTEN -nP 2>/dev/null
|
||||
```
|
||||
@@ -0,0 +1,248 @@
|
||||
# Python Debugging
|
||||
|
||||
Covers CPython 3.9+, pytest, asyncio, Django, FastAPI. Setup commands, attach mechanisms, state-query patterns, gotchas, silent-failure signatures.
|
||||
|
||||
---
|
||||
|
||||
## Environment detection (Phase 0)
|
||||
|
||||
```bash
|
||||
# Which Python will actually run the code?
|
||||
which python; which python3
|
||||
python --version
|
||||
|
||||
# Is there a project env manager in play?
|
||||
ls poetry.lock uv.lock Pipfile.lock requirements*.txt .python-version 2>/dev/null
|
||||
|
||||
# Installed debuggers / profilers in this env?
|
||||
python -c 'import pdb, sys; print("pdb", "built-in"); print("python", sys.executable)'
|
||||
pip list 2>/dev/null | grep -iE '^(ipdb|pudb|debugpy|py-spy|memray|rich)\s'
|
||||
|
||||
# asyncio debug mode available?
|
||||
python -c 'import asyncio; print(asyncio.__version__)'
|
||||
```
|
||||
|
||||
**Wrapper gotchas** (these change how flags propagate):
|
||||
|
||||
- `poetry run python ...` — args after `python` are fine; args before `poetry run` go to poetry, not python
|
||||
- `uv run python ...` — similar; prefer `uv run -- python -X dev` if flags collide
|
||||
- `pipenv run` — same story
|
||||
- `./manage.py <cmd>` (Django) — shebang resolution; make sure it points to the right venv
|
||||
- `pytest` — loads `conftest.py` at collection; breakpoints inside collection need `pytest --pdb-trace` not `--pdb`
|
||||
|
||||
---
|
||||
|
||||
## The four ways to attach
|
||||
|
||||
| Method | When to use | Command |
|
||||
|---|---|---|
|
||||
| **`breakpoint()` inline** (Python 3.7+) | You can edit the source and restart. Most reliable. | Add `breakpoint()` to source. Run normally. It invokes `pdb` by default. |
|
||||
| **`python -m pdb <script>`** | No source edit desired. Breaks on entry. | `python -m pdb script.py arg1` |
|
||||
| **post-mortem `pdb.pm()`** | Exception already happened, you want to inspect state | In an exception-caught REPL: `import pdb; pdb.pm()` after the exception propagates |
|
||||
| **debugpy (remote / IDE)** | IDE attach, remote host, containerized process | `python -m debugpy --listen 5678 --wait-for-client script.py` then attach from VS Code / PyCharm |
|
||||
|
||||
### Prefer `ipdb` or `pudb` over plain `pdb` when available
|
||||
|
||||
- **ipdb** — drop-in replacement with tab completion, syntax highlighting. `pip install ipdb`, then `PYTHONBREAKPOINT=ipdb.set_trace` or use `import ipdb; ipdb.set_trace()`.
|
||||
- **pudb** — full-screen TUI debugger, much faster to navigate stack/locals. `pip install pudb`, then `PYTHONBREAKPOINT=pudb.set_trace`.
|
||||
|
||||
### Control `breakpoint()` globally
|
||||
|
||||
```bash
|
||||
# Use ipdb instead of pdb
|
||||
export PYTHONBREAKPOINT=ipdb.set_trace
|
||||
|
||||
# Disable all breakpoint() calls (useful to ship without removing them)
|
||||
export PYTHONBREAKPOINT=0
|
||||
```
|
||||
|
||||
**Journal this env var** — unset at Phase 9.
|
||||
|
||||
---
|
||||
|
||||
## pdb / ipdb essentials
|
||||
|
||||
At a `(Pdb)` or `ipdb>` prompt:
|
||||
|
||||
```
|
||||
l list source around current line
|
||||
ll list whole function
|
||||
s step into
|
||||
n step over (next)
|
||||
r step out (return)
|
||||
c continue
|
||||
b list breakpoints
|
||||
b <line> breakpoint at line
|
||||
b <func> breakpoint at function
|
||||
cl <n> clear breakpoint n
|
||||
w where (backtrace)
|
||||
u / d move up/down the stack
|
||||
a args of current frame
|
||||
p <expr> print expression
|
||||
pp <expr> pretty-print
|
||||
!<stmt> execute Python statement (e.g. !x = 5)
|
||||
interact drop into a full Python REPL with current frame's locals
|
||||
q quit (aborts the program)
|
||||
```
|
||||
|
||||
**`interact` is underused** — it gives you a full IPython-esque REPL with all locals available. Faster than typing `p` for 20 things.
|
||||
|
||||
---
|
||||
|
||||
## pytest-specific debugging
|
||||
|
||||
```bash
|
||||
# Enter pdb on first failure
|
||||
pytest --pdb
|
||||
|
||||
# Enter pdb at the START of each test (not on failure)
|
||||
pytest --trace
|
||||
|
||||
# Run only the failing test, with -s to show print output
|
||||
pytest --pdb -x -s path/to/test.py::test_name
|
||||
|
||||
# Collect-time debugging (for problems in conftest.py / fixture setup)
|
||||
pytest --pdb-trace
|
||||
|
||||
# Disable capture for this test (so breakpoint prompt is visible)
|
||||
pytest -s
|
||||
```
|
||||
|
||||
**Common failure**: `breakpoint()` hangs inside a pytest test — that's because pytest captures stdout/stderr by default. Always add `-s` when debugging with breakpoints inside pytest.
|
||||
|
||||
---
|
||||
|
||||
## asyncio gotchas
|
||||
|
||||
Async is where most Python debug sessions go sideways. Know these before attaching.
|
||||
|
||||
### Breakpoints inside coroutines
|
||||
|
||||
`breakpoint()` works inside an async function, but stepping into another coroutine from `pdb` is awkward. Two techniques:
|
||||
|
||||
```python
|
||||
async def handler():
|
||||
result = await some_async_fn() # add breakpoint ABOVE, not inside, when possible
|
||||
breakpoint()
|
||||
return result
|
||||
```
|
||||
|
||||
Inside the breakpoint, to inspect a coroutine without actually advancing time:
|
||||
```
|
||||
!import asyncio
|
||||
!loop = asyncio.get_event_loop()
|
||||
!task = asyncio.ensure_future(some_async_fn())
|
||||
# Now inspect task state, don't await it
|
||||
p task
|
||||
```
|
||||
|
||||
### PYTHONASYNCIODEBUG
|
||||
|
||||
Enable before running the process:
|
||||
|
||||
```bash
|
||||
PYTHONASYNCIODEBUG=1 python script.py
|
||||
```
|
||||
|
||||
Surfaces: coroutines that were never awaited, slow callbacks, unhandled task exceptions. **Always turn this on** if the bug is timing- or async-related.
|
||||
|
||||
### `asyncio.gather` swallows the first exception
|
||||
|
||||
By default, `asyncio.gather(t1, t2)` raises the first exception and cancels the rest. If you need all exceptions, use `gather(..., return_exceptions=True)`.
|
||||
|
||||
### Unhandled task exceptions are silent
|
||||
|
||||
```python
|
||||
async def main():
|
||||
task = asyncio.create_task(broken_coroutine())
|
||||
# If task raises and we never await it, the exception is eaten at gc time
|
||||
await asyncio.sleep(10)
|
||||
```
|
||||
|
||||
To catch these, set `loop.set_exception_handler(...)` or upgrade to Python 3.12+ which warns louder by default.
|
||||
|
||||
---
|
||||
|
||||
## debugpy — remote / IDE / container attach
|
||||
|
||||
Listen and wait for attach:
|
||||
|
||||
```bash
|
||||
python -m debugpy --listen 0.0.0.0:5678 --wait-for-client script.py
|
||||
```
|
||||
|
||||
Attach from VS Code:
|
||||
```json
|
||||
// .vscode/launch.json
|
||||
{
|
||||
"name": "attach",
|
||||
"type": "python",
|
||||
"request": "attach",
|
||||
"connect": { "host": "localhost", "port": 5678 }
|
||||
}
|
||||
```
|
||||
|
||||
Inside the code, programmatic attach point:
|
||||
```python
|
||||
import debugpy
|
||||
debugpy.listen(5678)
|
||||
debugpy.wait_for_client() # blocks until attached
|
||||
debugpy.breakpoint() # programmatic breakpoint
|
||||
```
|
||||
|
||||
**Journal**: the port (5678) and the listener file — unset at Phase 9.
|
||||
|
||||
---
|
||||
|
||||
## Sampling profilers for "why is it slow / stuck"
|
||||
|
||||
When the problem is performance or a hang (not a crash), don't attach pdb — it alters timing. Use a sampling profiler that attaches to the running process:
|
||||
|
||||
```bash
|
||||
# py-spy — production-safe, zero code change, works on running process
|
||||
py-spy top --pid <pid> # live top-like view
|
||||
py-spy record -o profile.svg --pid <pid> # flamegraph
|
||||
py-spy dump --pid <pid> # stack traces of all threads right now
|
||||
|
||||
# memray — memory allocation tracking
|
||||
memray run script.py
|
||||
memray flamegraph output.bin
|
||||
memray stats output.bin
|
||||
```
|
||||
|
||||
`py-spy dump` on a stuck process is often enough to find the hung call — no breakpoints needed.
|
||||
|
||||
---
|
||||
|
||||
## Silent-failure patterns in Python
|
||||
|
||||
Add these to Phase 8's silent-failure check:
|
||||
|
||||
| Pattern | Why it's silent |
|
||||
|---|---|
|
||||
| `except Exception: pass` or `except: pass` | Catches and discards every error including KeyboardInterrupt |
|
||||
| `logging.exception(...)` in a logger with no handlers | "Logs" but actually writes nowhere |
|
||||
| `asyncio.create_task(coro)` without storing the task | Task GC'd before completion, exception swallowed |
|
||||
| `return x.get("key")` where key is missing | Returns None silently, caller often doesn't check |
|
||||
| `subprocess.run(..., check=False)` with ignored returncode | Non-zero exit treated as success |
|
||||
| Django `transaction.atomic()` inside a broader `except` | Rolls back silently |
|
||||
| `contextlib.suppress(Exception)` | Explicit silencer; easy to leave wider than intended |
|
||||
| `queue.get(block=False)` with `except queue.Empty: pass` | Polling that silently drops the work |
|
||||
|
||||
---
|
||||
|
||||
## Phase 9 cleanup specifics
|
||||
|
||||
```bash
|
||||
# Remove breakpoint() / ipdb / pudb lines from source
|
||||
git diff | grep -E '(breakpoint\(\)|import ipdb|import pudb|import pdb; pdb\.set_trace)'
|
||||
# If the above has output, revert those files:
|
||||
git checkout <file>
|
||||
|
||||
# Unset the global breakpoint override
|
||||
unset PYTHONBREAKPOINT
|
||||
|
||||
# Kill any leftover debugpy listeners
|
||||
pkill -f 'debugpy' || true
|
||||
lsof -iTCP:5678 -sTCP:LISTEN -nP 2>/dev/null # confirm free
|
||||
```
|
||||
@@ -0,0 +1,234 @@
|
||||
# Rust Debugging
|
||||
|
||||
Covers `cargo`, `tokio`, panics, and the fact that you usually don't actually need a debugger — Rust's type system, `dbg!`, and logging cover 80% of sessions faster than gdb would.
|
||||
|
||||
---
|
||||
|
||||
## Environment detection (Phase 0)
|
||||
|
||||
```bash
|
||||
cargo --version
|
||||
rustc --version
|
||||
cat rust-toolchain.toml rust-toolchain 2>/dev/null
|
||||
cat Cargo.toml | head -30
|
||||
|
||||
# Debuggers available
|
||||
which rust-gdb
|
||||
which rust-lldb
|
||||
which lldb
|
||||
|
||||
# Async infrastructure
|
||||
grep -E '"(tokio|async-std|smol)"' Cargo.toml
|
||||
|
||||
# Profile flags
|
||||
grep -E '^\[profile' Cargo.toml
|
||||
```
|
||||
|
||||
**The default `cargo run` builds with `dev` profile** which includes debug symbols. `cargo run --release` strips them. For debugging, stay in dev unless the bug only manifests under optimization.
|
||||
|
||||
---
|
||||
|
||||
## The Rust debugging hierarchy (use in this order)
|
||||
|
||||
Rust's ecosystem has a specific order that's faster than reaching for gdb first:
|
||||
|
||||
1. **`dbg!(expr)` macro** — for a single value at a specific spot. Prints file:line + value, returns the value unchanged so you can inline it. Faster than a debugger for 60% of bugs.
|
||||
2. **`RUST_LOG=trace` with `tracing` / `env_logger`** — for flow and state across an operation. Zero code change in dev-time.
|
||||
3. **`RUST_BACKTRACE=1` / `=full`** — for crashes. Almost always sufficient; you rarely need a live debugger for a panic.
|
||||
4. **`rust-gdb` / `rust-lldb`** — when you need to pause execution and inspect memory, especially for unsafe code or FFI.
|
||||
5. **`tokio-console`** — for async deadlocks, stuck tasks, hot loops.
|
||||
6. **`cargo-expand`** — when a macro is doing something weird.
|
||||
|
||||
Reach for the lightest tool that answers the hypothesis.
|
||||
|
||||
---
|
||||
|
||||
## `dbg!` — the underused macro
|
||||
|
||||
```rust
|
||||
let x = 5;
|
||||
let y = dbg!(x * 2); // prints: [src/main.rs:2] x * 2 = 10
|
||||
```
|
||||
|
||||
Inside a complex expression:
|
||||
```rust
|
||||
let total = items.iter().filter(|i| i.active).map(|i| dbg!(i.cost)).sum::<u64>();
|
||||
```
|
||||
|
||||
Multiple values at once:
|
||||
```rust
|
||||
dbg!(&user, &request, elapsed.as_millis());
|
||||
```
|
||||
|
||||
`dbg!` writes to stderr, so it won't corrupt stdout-based pipelines. **Journal each `dbg!` you add**; revert at Phase 9.
|
||||
|
||||
---
|
||||
|
||||
## `RUST_LOG` for flow-level debugging
|
||||
|
||||
If the codebase uses `tracing` or `env_logger`:
|
||||
|
||||
```bash
|
||||
RUST_LOG=debug cargo run
|
||||
RUST_LOG=trace cargo run # very verbose
|
||||
RUST_LOG=my_crate=trace,hyper=info cargo run # per-module level
|
||||
RUST_LOG=debug,tokio=off cargo run # silence noisy crates
|
||||
```
|
||||
|
||||
For `tracing`-based apps, instrument with spans:
|
||||
```rust
|
||||
#[tracing::instrument]
|
||||
fn handle_request(req: &Request) -> Response { ... }
|
||||
```
|
||||
|
||||
This gives you structured per-call entry/exit logs with args and timing, zero additional code in the body.
|
||||
|
||||
---
|
||||
|
||||
## `RUST_BACKTRACE` for panics
|
||||
|
||||
```bash
|
||||
RUST_BACKTRACE=1 cargo run # backtrace on panic
|
||||
RUST_BACKTRACE=full cargo run # include libstd/tokio frames
|
||||
```
|
||||
|
||||
The panic itself usually tells you the file:line. The backtrace tells you how it got there. Between the two, most crash bugs are solved without a debugger.
|
||||
|
||||
---
|
||||
|
||||
## rust-gdb / rust-lldb
|
||||
|
||||
### Launch
|
||||
|
||||
```bash
|
||||
# Build with debug symbols (default dev profile)
|
||||
cargo build
|
||||
|
||||
# Attach gdb wrapper (applies Rust type pretty-printers)
|
||||
rust-gdb ./target/debug/my_binary
|
||||
# Or lldb:
|
||||
rust-lldb ./target/debug/my_binary
|
||||
|
||||
# With args
|
||||
rust-gdb --args ./target/debug/my_binary arg1 arg2
|
||||
|
||||
# Attach to running process
|
||||
rust-gdb -p $(pgrep my_binary)
|
||||
```
|
||||
|
||||
### Breakpoints
|
||||
|
||||
Rust symbols are mangled. Use either:
|
||||
|
||||
```
|
||||
(gdb) b main # main function
|
||||
(gdb) b my_crate::module::function # canonical path
|
||||
(gdb) b src/handler.rs:42 # file:line
|
||||
(gdb) info functions my_function # find mangled name
|
||||
```
|
||||
|
||||
### State inspection
|
||||
|
||||
```
|
||||
(gdb) p x # print value (uses Rust pretty-printer for Vec, Option, HashMap, etc.)
|
||||
(gdb) p *ptr # deref
|
||||
(gdb) info locals # all locals in current frame
|
||||
(gdb) info args # function args
|
||||
(gdb) bt # backtrace
|
||||
(gdb) frame <n> # switch to stack frame n
|
||||
(gdb) watch my_var # stop when my_var changes
|
||||
(gdb) rbreak regex # breakpoint all functions matching regex
|
||||
```
|
||||
|
||||
**Pair with pwndbg** for better layout on native bugs — see [tools/pwndbg.md](../tools/pwndbg.md). Pwndbg works with rust-gdb too.
|
||||
|
||||
---
|
||||
|
||||
## tokio-console — async task debugging
|
||||
|
||||
For tokio-based async apps, this is essential when tasks are stuck or leaking.
|
||||
|
||||
```bash
|
||||
# Add tokio-console instrumentation to the target binary
|
||||
# In Cargo.toml:
|
||||
# [dependencies]
|
||||
# console-subscriber = "0.2"
|
||||
|
||||
# In main.rs:
|
||||
# console_subscriber::init();
|
||||
|
||||
# Build with tokio_unstable:
|
||||
RUSTFLAGS="--cfg tokio_unstable" cargo run
|
||||
|
||||
# In another terminal:
|
||||
tokio-console # connects to default port 6669
|
||||
```
|
||||
|
||||
Shows live tasks, their state, wake counts, poll durations, parent tasks. The single fastest way to find "why is my async thing stuck".
|
||||
|
||||
---
|
||||
|
||||
## cargo-expand — when a macro is suspect
|
||||
|
||||
```bash
|
||||
cargo install cargo-expand
|
||||
cargo expand # expand all macros in the crate
|
||||
cargo expand my::module::path # scope to one item
|
||||
```
|
||||
|
||||
If you suspect a macro (especially `#[derive]`, `#[tokio::main]`, `#[async_trait]`) is generating code that doesn't match your mental model, this shows you exactly what the compiler sees.
|
||||
|
||||
---
|
||||
|
||||
## Release-build gotcha
|
||||
|
||||
```bash
|
||||
cargo build --release # no debug symbols by default
|
||||
```
|
||||
|
||||
If the bug only shows up in `--release`:
|
||||
|
||||
```toml
|
||||
# Cargo.toml
|
||||
[profile.release]
|
||||
debug = true # add symbols, keep optimizations
|
||||
```
|
||||
|
||||
Now `rust-gdb ./target/release/my_binary` works on release builds. This is required when optimization-enabled codegen bugs (inlining, LLVM folding) are suspected.
|
||||
|
||||
---
|
||||
|
||||
## Silent-failure patterns in Rust
|
||||
|
||||
| Pattern | Why it's silent |
|
||||
|---|---|
|
||||
| `.unwrap_or_default()` | Masks errors as the zero value |
|
||||
| `.unwrap_or(fallback)` | Same, with a specific fallback |
|
||||
| `let _ = fallible_operation()` | Explicitly discards the Result, no compiler warning |
|
||||
| `if let Ok(x) = ... { use(x); } // no else` | Silent on Err |
|
||||
| `.ok()` chaining | Converts Result to Option, throwing the error away |
|
||||
| Panic inside a tokio task not `.await`ed | Task dies silently; runtime usually logs but it's quiet if logs are off |
|
||||
| `eprintln!` that goes to a redirected-null stderr | Looks like nothing happened |
|
||||
| `Drop` impl that panics under specific condition | Double-panic aborts process silently if no logging configured |
|
||||
|
||||
---
|
||||
|
||||
## Phase 9 cleanup specifics
|
||||
|
||||
```bash
|
||||
# Revert dbg! macro additions
|
||||
git diff | grep -E 'dbg!\('
|
||||
# For any file with dbg! additions:
|
||||
git checkout <file>
|
||||
|
||||
# Unset env vars
|
||||
unset RUST_LOG RUST_BACKTRACE
|
||||
|
||||
# Kill any rust-gdb/lldb sessions
|
||||
pkill -f 'rust-gdb' || true
|
||||
pkill -f 'rust-lldb' || true
|
||||
pkill -f '^lldb ' || true
|
||||
|
||||
# tokio-console binds 6669 by default
|
||||
lsof -iTCP:6669 -sTCP:LISTEN -nP 2>/dev/null
|
||||
```
|
||||
@@ -0,0 +1,212 @@
|
||||
# Ghidra — Decompile Binaries Into Readable C
|
||||
|
||||
**https://github.com/NationalSecurityAgency/ghidra**
|
||||
|
||||
Ghidra is the NSA's open-source reverse-engineering suite. Its defining feature is a **decompiler** that turns machine code back into readable C. For any binary you don't have source for, this is the correct starting point — not `strings`, not hex-staring, not `objdump -d`.
|
||||
|
||||
**Use Ghidra when**: third-party closed-source libs, malware analysis, vendored binaries whose behavior contradicts docs, CTF challenges, firmware, any time you need to read compiled code.
|
||||
|
||||
---
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
# macOS
|
||||
brew install --cask ghidra
|
||||
# OR download the release ZIP from the repo and ./ghidraRun
|
||||
|
||||
# Linux
|
||||
# Download from https://github.com/NationalSecurityAgency/ghidra/releases
|
||||
# Requires JDK 21+
|
||||
./ghidraRun
|
||||
|
||||
# Dependency
|
||||
java -version # must be 21+
|
||||
```
|
||||
|
||||
Ghidra is a Java Swing app. Looks dated, works well.
|
||||
|
||||
---
|
||||
|
||||
## First-time workflow (memorize this — it's not obvious)
|
||||
|
||||
1. **Start Ghidra**: `ghidraRun`
|
||||
2. **Create a project**: File → New Project → Non-Shared → name it `debug-<binary-name>` (journal this path so you can rm it at Phase 9 if disposable).
|
||||
3. **Import the binary**: File → Import File → pick your target. Accept default format detection.
|
||||
4. **Double-click the imported binary** in the project listing. Ghidra asks to analyze it — say **yes**, accept defaults for the first pass. This takes anywhere from seconds (small binary) to tens of minutes (large binary).
|
||||
5. **Once analysis completes**, you're in the CodeBrowser view.
|
||||
|
||||
Two panels you'll use 95% of the time:
|
||||
|
||||
- **Listing** (middle) — the disassembly with Ghidra's inferred labels/types.
|
||||
- **Decompiler** (right) — the reconstructed C. The real value.
|
||||
|
||||
---
|
||||
|
||||
## Finding the right function fast
|
||||
|
||||
Don't try to read the whole binary. Use these to narrow:
|
||||
|
||||
### Symbol Tree (left panel)
|
||||
|
||||
- `Functions` — all detected functions. Stripped binaries show `FUN_00401234` (address-named); unstripped show actual names.
|
||||
- `Imports` — dynamically-linked functions. Great for "does this binary call `system()`, `strcpy`, `curl_easy_perform`?"
|
||||
- `Exports` — if it's a library.
|
||||
|
||||
Click to jump. The Decompiler updates instantly.
|
||||
|
||||
### String search
|
||||
|
||||
```
|
||||
Search → For Strings
|
||||
```
|
||||
|
||||
Produces a list of all strings. Right-click a string → `References` → `Show References to Address`. Jumps to code that references it. **This is how you find which function handles the error message you saw at runtime.**
|
||||
|
||||
### Memory search for bytes
|
||||
|
||||
```
|
||||
Search → Memory
|
||||
```
|
||||
|
||||
Search hex or text. Useful for known magic bytes, file-format signatures, constants.
|
||||
|
||||
### Cross-references (XREF)
|
||||
|
||||
Right-click any function / address → `References → Find References to`. Shows every place that calls it. Walk the call graph backward from interesting functions.
|
||||
|
||||
---
|
||||
|
||||
## Making the decompiler's output readable
|
||||
|
||||
Ghidra's decompiler is good but needs hints. These three actions dramatically improve its output:
|
||||
|
||||
### 1. Rename variables
|
||||
|
||||
Click a variable in the Decompiler view → press `L` → type a better name. Ghidra propagates the rename across all uses.
|
||||
|
||||
### 2. Set types
|
||||
|
||||
A variable that looks like `undefined4` or `void *` is unhelpful. Click it → press `Ctrl+L` → set type (e.g. `int`, `char *`, `struct my_header *`).
|
||||
|
||||
For pointers to structs from headers you have, use:
|
||||
```
|
||||
File → Parse C Source → paste header file → auto-creates struct types
|
||||
```
|
||||
|
||||
Then assign the struct type to the pointer. Ghidra resolves field accesses immediately.
|
||||
|
||||
### 3. Retype function signatures
|
||||
|
||||
Click the function name in the Decompiler → press `F` (Edit Function Signature) → set return type + argument types. This propagates through callers.
|
||||
|
||||
Do these three actions on the 3-5 most relevant functions and the decompiler output becomes near-source-readable.
|
||||
|
||||
---
|
||||
|
||||
## Patterns for specific bug types
|
||||
|
||||
### Looking for integer overflow / buffer overflow
|
||||
|
||||
```
|
||||
Listing: look for
|
||||
- LEA → CMP patterns on sizes
|
||||
- memcpy/strcpy/sprintf with non-constant sizes
|
||||
Decompiler: look for
|
||||
- arithmetic on size_t without bounds check
|
||||
- `+ user_input` in a length calculation
|
||||
```
|
||||
|
||||
### Looking for a missing auth check
|
||||
|
||||
Navigate from the handler entry (found via Strings or Imports) and check the control flow:
|
||||
|
||||
```
|
||||
Decompiler: does the function return early / jump to error handler when some flag is not set?
|
||||
If the check is absent, that's the bug.
|
||||
```
|
||||
|
||||
### Looking for hardcoded URLs / keys / paths
|
||||
|
||||
```
|
||||
Search → For Strings → filter `http:` / `https:` / `/etc/` / `bearer ` / `api_key`
|
||||
```
|
||||
|
||||
### Looking for dispatch / plugin loading
|
||||
|
||||
```
|
||||
Imports → dlopen, LoadLibrary, dlsym, GetProcAddress
|
||||
```
|
||||
|
||||
The strings referenced near those calls are often plugin names.
|
||||
|
||||
---
|
||||
|
||||
## Scripting (headless Ghidra)
|
||||
|
||||
When you need to automate analysis across many binaries, or repeat a workflow:
|
||||
|
||||
```bash
|
||||
# Headless analyzer
|
||||
$GHIDRA_INSTALL_DIR/support/analyzeHeadless \
|
||||
<project-dir> <project-name> \
|
||||
-import <binary> \
|
||||
-postScript <script.py or script.java>
|
||||
```
|
||||
|
||||
Ghidra supports Python 3 scripts (via Jython-compatible API) and Java. Useful scripts:
|
||||
|
||||
- Dump all function signatures to JSON
|
||||
- Find all calls to `system()` with constant arguments
|
||||
- Auto-rename FUN_xxx based on heuristics (string refs, call patterns)
|
||||
|
||||
The community has a large collection: https://github.com/NationalSecurityAgency/ghidra/tree/master/Ghidra/Features/Base/ghidra_scripts
|
||||
|
||||
---
|
||||
|
||||
## Bookmarks + Notes
|
||||
|
||||
Ghidra has built-in bookmarks and comments. Use them as your journal inside the project:
|
||||
|
||||
- Right-click an address → `Set Bookmark` → tag as `Note`. Attach a description.
|
||||
- Right-click → `Comments → Set EOL Comment` (shows up inline in decompiler).
|
||||
|
||||
Treat these as part of the journal. If you end up promoting this Ghidra project (keeping it after the debug session), the comments become durable documentation.
|
||||
|
||||
---
|
||||
|
||||
## Gotchas
|
||||
|
||||
- **Large binaries need more Java heap.** Edit `support/launch.properties` and bump `VMARGS=-Xmx8G` (default is often 2G, too small).
|
||||
- **Save often.** Ghidra's autosave is not instant; a crash loses uncommitted analysis.
|
||||
- **Decompiler has timeouts.** For complex functions it may give up and print `/* WARNING: ... */`. Increase timeout in `Edit → Tool Options → Decompiler`.
|
||||
- **Archs beyond x86/ARM/MIPS** sometimes need Sleigh processor module tweaks. Rare but possible.
|
||||
- **Signed vs unsigned decompilation** is frequently wrong. Manually retype when integer behavior matters.
|
||||
|
||||
---
|
||||
|
||||
## When Ghidra is NOT the right tool
|
||||
|
||||
- You have the source. Go read it.
|
||||
- Bug is in your own recently-compiled binary. Rebuild with `-g` and use gdb/pwndbg (see [pwndbg.md](pwndbg.md)).
|
||||
- You just need to know which libs a binary links. `ldd` / `otool -L` / `readelf -d` are faster.
|
||||
- You just need strings. `strings -n 8` is faster.
|
||||
|
||||
Ghidra is the right tool when you need to **read the logic** of a binary you don't have source for.
|
||||
|
||||
---
|
||||
|
||||
## Phase 9 cleanup specifics
|
||||
|
||||
```bash
|
||||
# If you created a scratch project just for this debug session, journal path and remove:
|
||||
# (the journal should have the exact path)
|
||||
# Example:
|
||||
ls ~/ghidra-projects/ 2>/dev/null
|
||||
# rm -rf ~/ghidra-projects/debug-<binary-name>
|
||||
|
||||
# If you promoted the project (kept it), it's not cleanup — note it in the final summary to the user
|
||||
|
||||
# Kill any running Ghidra headless processes
|
||||
pkill -f 'analyzeHeadless' || true
|
||||
```
|
||||
@@ -0,0 +1,194 @@
|
||||
# Playwright CLI — Browser QA That Actually Drives a Browser
|
||||
|
||||
**https://playwright.dev/ · https://github.com/microsoft/playwright**
|
||||
|
||||
For any browser-served web UI bug, this is the correct tool. Not curl. Not imagination. Not a headless HTTP library. A real browser with a real rendering engine, real JS execution, real cookies, real service workers, real viewport.
|
||||
|
||||
**In Phase 8 Manual QA for browser products, using Playwright is not optional.** Curl cannot catch: CSS that breaks at specific viewport widths, hydration mismatches, client-side router bugs, cookie/session interactions, service-worker caching, JS-triggered navigations. All of those are common bug classes. Drive a browser.
|
||||
|
||||
> Note: `microsoft/playwright-cli` is the legacy repo; the current tooling lives in `@playwright/test` (npm) and `playwright` (pip), which include the `playwright` CLI. Use those — the legacy `playwright-cli` package is deprecated.
|
||||
|
||||
---
|
||||
|
||||
## When to reach for Playwright
|
||||
|
||||
| Bug symptom | Use Playwright? |
|
||||
|---|---|
|
||||
| Form submit produces wrong result | ✅ — Playwright drives the form exactly as a user does |
|
||||
| Page blank in prod, fine locally | ✅ — hydration/env differences need a real browser |
|
||||
| CSS looks wrong at a specific width | ✅ — use `--viewport-size` |
|
||||
| Click doesn't fire / wrong handler | ✅ — Playwright fires real DOM events |
|
||||
| Flash of unstyled content / loading glitch | ✅ — use trace viewer to see frames |
|
||||
| API returns wrong data | ❌ — use curl, this isn't a browser bug |
|
||||
| Backend returns wrong status code | ❌ — use curl |
|
||||
| Client hits a URL that returns 500 | ✅ but also ❌ — Playwright shows the call + response + failure effect on UI |
|
||||
|
||||
---
|
||||
|
||||
## Install (per-project)
|
||||
|
||||
Playwright installs browser binaries separately from the npm package.
|
||||
|
||||
```bash
|
||||
# In the project
|
||||
npm init playwright@latest # interactive; picks TS/JS + browsers + config
|
||||
# Or if Playwright is already a dep:
|
||||
npx playwright install # downloads browsers
|
||||
npx playwright install chromium # just chromium
|
||||
npx playwright install --with-deps # also installs OS deps (Linux)
|
||||
```
|
||||
|
||||
Python:
|
||||
```bash
|
||||
pip install playwright
|
||||
playwright install
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## The four things you'll actually use
|
||||
|
||||
### 1. `codegen` — record a session, generate the script
|
||||
|
||||
The fastest way to create a repro. Opens a real browser; your clicks / typing become a Playwright script you can paste into a test.
|
||||
|
||||
```bash
|
||||
npx playwright codegen https://your-app.local
|
||||
npx playwright codegen --viewport-size=375,667 https://your-app.local # iPhone SE size
|
||||
npx playwright codegen --device="iPhone 14" https://your-app.local
|
||||
```
|
||||
|
||||
Click / type / navigate in the browser; watch the script build in the side panel. Copy the generated script into your journal as the repro for Phase 8.
|
||||
|
||||
### 2. A one-shot Playwright script — reproduce + capture
|
||||
|
||||
Usually the Phase 8 QA artifact. Save to `/tmp/debug-repro.spec.ts` (journal it):
|
||||
|
||||
```ts
|
||||
// /tmp/debug-repro.spec.ts
|
||||
import { test, expect } from '@playwright/test';
|
||||
|
||||
test('refinement chat shows non-empty response when env var set', async ({ page }) => {
|
||||
await page.goto('http://localhost:3000/chat');
|
||||
await page.fill('textarea[name="message"]', 'Add a logging step');
|
||||
await page.click('button[type=submit]');
|
||||
|
||||
// Wait for the response to appear (not just the spinner to disappear)
|
||||
const response = page.locator('[data-testid="assistant-reply"]');
|
||||
await expect(response).toBeVisible({ timeout: 30_000 });
|
||||
await expect(response).not.toBeEmpty();
|
||||
|
||||
// Capture evidence
|
||||
await page.screenshot({ path: '/tmp/debug-after-fix.png', fullPage: true });
|
||||
console.log(await response.textContent());
|
||||
});
|
||||
```
|
||||
|
||||
Run it with tracing enabled for rich post-mortem:
|
||||
|
||||
```bash
|
||||
npx playwright test /tmp/debug-repro.spec.ts --trace on --headed
|
||||
```
|
||||
|
||||
### 3. `PWDEBUG=1` — step through the script with Playwright Inspector
|
||||
|
||||
```bash
|
||||
PWDEBUG=1 npx playwright test /tmp/debug-repro.spec.ts
|
||||
```
|
||||
|
||||
Opens the Playwright Inspector alongside the browser. You can step through Playwright actions, see the DOM state at each step, and edit selectors on the fly.
|
||||
|
||||
Use this when the script doesn't reproduce cleanly and you need to watch it run.
|
||||
|
||||
### 4. `show-trace` — post-mortem on a failed run
|
||||
|
||||
```bash
|
||||
npx playwright show-trace trace.zip
|
||||
# or from the test-results dir:
|
||||
npx playwright show-trace test-results/<test-name>/trace.zip
|
||||
```
|
||||
|
||||
Scrubs through a recorded session: timeline, DOM snapshot at each action, network, console, source. When a test failed on CI but passed locally, this is the single best artifact.
|
||||
|
||||
---
|
||||
|
||||
## Headless vs headed during debugging
|
||||
|
||||
Always add `--headed` when debugging. Headless browsers sometimes behave subtly differently (font rendering, viewport, media permissions). For QA evidence, run headed and screenshot.
|
||||
|
||||
```bash
|
||||
npx playwright test --headed
|
||||
npx playwright test --headed --project=chromium # pin the browser
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Catching the silent-failure patterns Playwright is good at
|
||||
|
||||
```ts
|
||||
// Toast that flashes and disappears
|
||||
page.on('console', msg => console.log('[browser console]', msg.type(), msg.text()));
|
||||
|
||||
// Unhandled page errors (uncaught exceptions in the page JS)
|
||||
page.on('pageerror', err => console.error('[page error]', err));
|
||||
|
||||
// Network failures — e.g., backend returned 500 but UI shows nothing
|
||||
page.on('response', async resp => {
|
||||
if (!resp.ok()) {
|
||||
console.warn(`[network ${resp.status()}] ${resp.url()} — ${await resp.text()}`);
|
||||
}
|
||||
});
|
||||
|
||||
// Request that never came back
|
||||
page.on('requestfailed', req => {
|
||||
console.error('[request failed]', req.url(), req.failure()?.errorText);
|
||||
});
|
||||
```
|
||||
|
||||
Add these listeners to the top of the debug script. They surface a lot of the "UI showed nothing" class of bug.
|
||||
|
||||
---
|
||||
|
||||
## Viewport and device emulation
|
||||
|
||||
CSS bugs that only appear at specific sizes, or layout bugs on mobile:
|
||||
|
||||
```ts
|
||||
// At test level
|
||||
test.use({ viewport: { width: 375, height: 667 } });
|
||||
|
||||
// Per-page
|
||||
await page.setViewportSize({ width: 375, height: 667 });
|
||||
|
||||
// Predefined devices
|
||||
import { devices } from '@playwright/test';
|
||||
test.use({ ...devices['iPhone 14'] });
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Gotchas
|
||||
|
||||
- **Wait for state, not for time.** `await page.waitForTimeout(2000)` is flaky. Use `await expect(locator).toBeVisible()` or `page.waitForResponse(urlPattern)`.
|
||||
- **Stale selectors re-resolve.** Playwright's locators re-find the element on each action, unlike Puppeteer's handles. Don't over-think it.
|
||||
- **Service workers persist across test runs in headed mode.** If you see cached behavior from a previous run, add `await context.clearCookies()` + clear storage before the test.
|
||||
- **Installing on CI requires `--with-deps`** on Linux images that lack the browser's shared-library deps.
|
||||
- **Parallel tests share a browser process by default**; if one test polls a debugger port, others may interfere. Use `workers: 1` for debugging.
|
||||
|
||||
---
|
||||
|
||||
## Phase 9 cleanup specifics
|
||||
|
||||
```bash
|
||||
# Remove trace files from debug runs
|
||||
rm -rf playwright-report/ test-results/ trace.zip
|
||||
|
||||
# Remove debug spec files from /tmp
|
||||
rm -f /tmp/debug-*.spec.ts
|
||||
|
||||
# Remove screenshot captures
|
||||
rm -f /tmp/debug-*.png
|
||||
|
||||
# If you installed browsers just for this session (rare):
|
||||
# Don't remove them — they're useful for future sessions. They live in ~/Library/Caches/ms-playwright (macOS) or ~/.cache/ms-playwright (Linux).
|
||||
```
|
||||
@@ -0,0 +1,263 @@
|
||||
# pwndbg — GDB With the Useful Views Always On
|
||||
|
||||
**https://github.com/pwndbg/pwndbg**
|
||||
|
||||
pwndbg is a GDB plugin that turns GDB into something humans can actually use for binary debugging. It's strictly a superset of plain GDB — every vanilla GDB command still works, and pwndbg adds views and commands that make you productive.
|
||||
|
||||
**If you'd reach for plain `gdb`, reach for pwndbg instead.** The only reason not to is if pwndbg isn't installed on the machine, and that's a 2-minute fix.
|
||||
|
||||
---
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
# macOS
|
||||
brew install pwndbg
|
||||
# Or from source:
|
||||
git clone https://github.com/pwndbg/pwndbg
|
||||
cd pwndbg && ./setup.sh
|
||||
|
||||
# Linux
|
||||
# Most distros: apt/dnf/pacman install pwndbg (check availability)
|
||||
# Or the same git + ./setup.sh
|
||||
|
||||
# Verify
|
||||
gdb --version
|
||||
gdb ./any-binary
|
||||
# At gdb prompt, you should see pwndbg banner + colorful context view
|
||||
```
|
||||
|
||||
Once installed, pwndbg auto-loads every time you start `gdb`. You don't source anything manually.
|
||||
|
||||
---
|
||||
|
||||
## The `context` view — the one feature that changes everything
|
||||
|
||||
Plain GDB: you run `info registers`, then `bt`, then `x/10xw $rsp`, then `disas`. Four commands to see what's going on.
|
||||
|
||||
pwndbg: `context` (or it auto-shows at every break). One command. Everything on screen:
|
||||
|
||||
```
|
||||
──── registers ────
|
||||
RAX 0x0
|
||||
RBX 0x7ffffffde158
|
||||
RCX 0x7fffff7abf10
|
||||
...
|
||||
──── disasm ────
|
||||
► 0x401234 mov rdi, rax
|
||||
0x401237 call 0x401190
|
||||
...
|
||||
──── stack ────
|
||||
00:0000│ rsp 0x7ffffffde0a0 → 0x7fffff7c4000
|
||||
01:0008│ 0x7ffffffde0a8 → 0x0
|
||||
...
|
||||
──── backtrace ────
|
||||
► f 0 0x401234 parse_input+0x3c
|
||||
f 1 0x401180 main+0x120
|
||||
f 2 0x7fffff7a5083 __libc_start_main+0xf3
|
||||
```
|
||||
|
||||
You always know where you are, what the CPU state is, what's on the stack, and how you got here. This is why pwndbg is the default.
|
||||
|
||||
---
|
||||
|
||||
## Launch recipes
|
||||
|
||||
```bash
|
||||
# Debug an existing binary
|
||||
gdb ./target
|
||||
|
||||
# With args
|
||||
gdb --args ./target arg1 arg2
|
||||
|
||||
# Attach to a running process
|
||||
gdb -p $(pgrep target)
|
||||
|
||||
# With a core dump
|
||||
gdb ./target ./core
|
||||
|
||||
# Headless / remote (for automation or IDE attach)
|
||||
gdbserver :2345 ./target # on the target box
|
||||
gdb ./target # on your box
|
||||
(gdb) target remote <host>:2345
|
||||
```
|
||||
|
||||
At the pwndbg prompt:
|
||||
|
||||
---
|
||||
|
||||
## Essential commands (pwndbg additions)
|
||||
|
||||
### Layout / view
|
||||
|
||||
```
|
||||
context # reprint the context view (usually auto)
|
||||
context regs stack # only show registers + stack sections
|
||||
tel $rsp 20 # telescope — walk pointers at $rsp for 20 slots (KEY COMMAND)
|
||||
tel $rdi 10 # walk pointers at $rdi (e.g. to dump a struct)
|
||||
stack 20 # 20 entries of stack
|
||||
vmmap # virtual memory map of the process
|
||||
```
|
||||
|
||||
**`telescope` is pwndbg's killer command.** Given an address, it walks pointers recursively:
|
||||
```
|
||||
00:0000│ 0x7ffd... → 0x601010 (heap) → 0x2a (unknown, i.e. a number 42)
|
||||
01:0008│ 0x7ffd... → 0x7fff... (stack) → 'hello world'
|
||||
```
|
||||
This single view resolves 80% of "what is at this address" questions.
|
||||
|
||||
### Heap debugging
|
||||
|
||||
```
|
||||
heap # overview of chunks
|
||||
bins # tcache / fastbin / unsorted / smallbin / largebin state
|
||||
malloc_chunk <addr> # inspect a specific chunk
|
||||
find_fake_fast <addr> # (exploit context) find fake-fast overlap candidates
|
||||
vis_heap_chunks # visualize heap layout
|
||||
```
|
||||
|
||||
For use-after-free / double-free / heap overflow hypotheses, `heap` + `bins` is usually sufficient to see the corruption.
|
||||
|
||||
### Exploitation-adjacent (useful for bug understanding too)
|
||||
|
||||
```
|
||||
checksec # NX, PIE, RELRO, canary status
|
||||
rop --grep 'pop rdi' # find ROP gadgets
|
||||
nx # step over (aliased nicely)
|
||||
ni # step over single instruction
|
||||
si # step into single instruction
|
||||
```
|
||||
|
||||
### Search
|
||||
|
||||
```
|
||||
search -t byte 0x41 # find byte 0x41 anywhere in memory
|
||||
search -t string "admin" # find string
|
||||
search -p <addr> # find pointers to <addr>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Standard GDB commands still work
|
||||
|
||||
pwndbg doesn't replace GDB; it augments it. Everything you know still works:
|
||||
|
||||
```
|
||||
break main # breakpoint at function
|
||||
b *0x401234 # breakpoint at address
|
||||
b file.c:42 # breakpoint at file:line
|
||||
c # continue
|
||||
n # next (source-level step over)
|
||||
s # step (source-level step into)
|
||||
finish # step out
|
||||
info breakpoints # list breakpoints
|
||||
delete <n> # delete breakpoint
|
||||
watch <var> # break on write to variable
|
||||
rwatch <var> # break on read
|
||||
awatch <var> # break on access
|
||||
p <expr> # print expression
|
||||
p/x <expr> # print in hex
|
||||
x/20xw <addr> # examine 20 words as hex
|
||||
bt # backtrace
|
||||
frame <n> # switch frame
|
||||
info registers # registers (but `context` is better)
|
||||
disassemble <func> # disasm a function
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Python scripting inside GDB
|
||||
|
||||
pwndbg exposes a full Python API. Useful for automating observations across many breakpoints:
|
||||
|
||||
```python
|
||||
(gdb) python
|
||||
import gdb
|
||||
def on_break():
|
||||
frame = gdb.selected_frame()
|
||||
pc = frame.read_register('pc')
|
||||
print(f'hit at {hex(int(pc))}')
|
||||
# Dump args, locals, anything
|
||||
end
|
||||
```
|
||||
|
||||
Or scripted runs from outside:
|
||||
```bash
|
||||
gdb -batch -ex 'source script.gdb' -ex 'run' ./target
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Common workflows by bug type
|
||||
|
||||
### Segfault / crash
|
||||
|
||||
```bash
|
||||
gdb ./target
|
||||
(gdb) run <args>
|
||||
# ... crash ...
|
||||
(gdb) context # see the crash site
|
||||
(gdb) bt # how did we get here?
|
||||
(gdb) info registers # what state
|
||||
(gdb) tel $rsp 20 # what's on the stack
|
||||
```
|
||||
|
||||
### "Function returns wrong value"
|
||||
|
||||
```bash
|
||||
gdb ./target
|
||||
(gdb) break <function>
|
||||
(gdb) run <args>
|
||||
# At breakpoint:
|
||||
(gdb) finish # let it run to the return
|
||||
# pwndbg shows RAX (return value) in context
|
||||
```
|
||||
|
||||
### "Variable has unexpected value at point X"
|
||||
|
||||
```bash
|
||||
gdb ./target
|
||||
(gdb) break <point-X>
|
||||
(gdb) run
|
||||
# At breakpoint:
|
||||
(gdb) p <var> # its value
|
||||
(gdb) watch <var> # set a watchpoint — break when it changes
|
||||
(gdb) c # continue; next stop is where it was modified
|
||||
```
|
||||
|
||||
### "Memory corruption / heap bug"
|
||||
|
||||
```bash
|
||||
gdb ./target
|
||||
(gdb) run
|
||||
# Crash at free():
|
||||
(gdb) heap # heap state
|
||||
(gdb) bins # bin state — often shows corruption here
|
||||
(gdb) vis_heap_chunks # visualize
|
||||
(gdb) malloc_chunk <suspicious-addr>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Gotchas
|
||||
|
||||
- **`bt` looks weird on stripped binaries** — function names become offsets. Use Ghidra's function labels to map back (see [ghidra.md](ghidra.md)).
|
||||
- **PIE binaries have randomized base addresses.** Addresses you see in Ghidra are unslid; addresses in pwndbg are slid. The `vmmap` command shows the base, and pwndbg's `piebase` command gives you the offset.
|
||||
- **Optimized builds inline functions.** You'll set a breakpoint on `my_function` and it won't hit because the function was inlined. Either disable optimizations or break on callers.
|
||||
- **Stack canaries trigger `__stack_chk_fail`.** If you see that in a backtrace, the bug caused a stack-smash; look one frame up.
|
||||
|
||||
---
|
||||
|
||||
## Phase 9 cleanup specifics
|
||||
|
||||
```bash
|
||||
# Kill gdb / pwndbg sessions
|
||||
pkill -f 'gdb' || true
|
||||
pkill -f 'gdbserver' || true
|
||||
|
||||
# Remove core dumps generated during session
|
||||
rm -f ./core ./core.* ~/core.*
|
||||
|
||||
# Remove any scripted GDB files
|
||||
rm -f /tmp/debug-*.gdb
|
||||
```
|
||||
@@ -0,0 +1,265 @@
|
||||
# pwntools — Scripted Binary / Network Interaction
|
||||
|
||||
**https://docs.pwntools.com/en/stable/ · https://github.com/Gallopsled/pwntools**
|
||||
|
||||
pwntools is a Python framework for building reproducible interactions with binaries and network services. Originally built for CTF exploitation, it's the correct tool for any situation where you need:
|
||||
|
||||
- A crafted input sent to a binary or network service, repeatably
|
||||
- A "failing test" equivalent for a bug that only manifests with specific byte-level input
|
||||
- A fuzz harness
|
||||
- An exploit PoC
|
||||
- Anything where you're tempted to use `echo ... | ./binary` but need more control than shell allows
|
||||
|
||||
**Use pwntools for Phase 5 reproduction of binary bugs and Phase 7 tests against binaries.**
|
||||
|
||||
---
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
pip install pwntools
|
||||
# Or in a venv:
|
||||
python -m venv .venv && source .venv/bin/activate
|
||||
pip install pwntools
|
||||
|
||||
# Verify
|
||||
python -c 'from pwn import *; print("ok")'
|
||||
```
|
||||
|
||||
On some Linux distros you may need build deps: `apt install python3-dev libssl-dev`.
|
||||
|
||||
---
|
||||
|
||||
## The core API in five idioms
|
||||
|
||||
### 1. Process / Remote — the same interface
|
||||
|
||||
```python
|
||||
from pwn import *
|
||||
|
||||
# Local process
|
||||
p = process('./target')
|
||||
|
||||
# Remote service
|
||||
p = remote('example.com', 1337)
|
||||
|
||||
# SSH (tunnel to a remote process)
|
||||
shell = ssh('user', 'host', password='...')
|
||||
p = shell.process('./target', cwd='/tmp')
|
||||
|
||||
# Same methods on all of the above — this is the value proposition
|
||||
```
|
||||
|
||||
### 2. I/O — the only five methods you need
|
||||
|
||||
```python
|
||||
p.send(b'data') # send bytes
|
||||
p.sendline(b'data') # send bytes + \n
|
||||
p.recv(n) # receive up to n bytes
|
||||
p.recvuntil(b'> ') # receive until pattern (blocks)
|
||||
p.recvline() # receive until \n
|
||||
p.interactive() # hand control to your terminal (for manual exploration)
|
||||
|
||||
# Combined
|
||||
p.sendlineafter(b'prompt> ', b'payload')
|
||||
p.sendafter(b'key:', key)
|
||||
```
|
||||
|
||||
Timeouts:
|
||||
```python
|
||||
try:
|
||||
data = p.recvuntil(b'done', timeout=5)
|
||||
except pwnlib.exception.EOFError:
|
||||
print('process died')
|
||||
except TimeoutError:
|
||||
print('no response in 5s')
|
||||
```
|
||||
|
||||
### 3. context — set arch/OS once, tools align
|
||||
|
||||
```python
|
||||
context.binary = elf = ELF('./target') # auto-sets arch/os/endianness
|
||||
# or explicitly:
|
||||
context.update(arch='amd64', os='linux', endian='little', bits=64)
|
||||
```
|
||||
|
||||
After setting context, helpers like `asm()`, `disasm()`, `cyclic()`, and `ROP()` produce correct output for that target automatically.
|
||||
|
||||
### 4. ELF — parse without reverse-engineering by hand
|
||||
|
||||
```python
|
||||
elf = ELF('./target')
|
||||
|
||||
elf.symbols['main'] # address of main
|
||||
elf.plt['printf'] # address in PLT (dynamic linkage)
|
||||
elf.got['printf'] # GOT entry
|
||||
elf.address = 0x555555554000 # set base for PIE binaries
|
||||
elf.search(b'/bin/sh') # find string or bytes in the binary
|
||||
elf.functions['main'].address # same as elf.symbols['main']
|
||||
list(elf.functions)[:10] # first 10 function names
|
||||
```
|
||||
|
||||
For the libc that's linked:
|
||||
```python
|
||||
libc = ELF('/lib/x86_64-linux-gnu/libc.so.6')
|
||||
libc.symbols['system']
|
||||
```
|
||||
|
||||
### 5. cyclic — find offsets without counting
|
||||
|
||||
For "where exactly does user input reach this variable" bugs:
|
||||
|
||||
```python
|
||||
p = process('./target')
|
||||
p.sendline(cyclic(256)) # send a De Bruijn pattern
|
||||
# Crash occurs; note the crash value (e.g. RIP = 0x6161616c)
|
||||
offset = cyclic_find(0x6161616c) # returns 12 (or wherever in the pattern)
|
||||
# Now you know: byte 12 of your input lands at RIP
|
||||
```
|
||||
|
||||
Saves an hour of "pad by N bytes then check" iteration.
|
||||
|
||||
---
|
||||
|
||||
## Logging during debug
|
||||
|
||||
pwntools logs output by default. Configure level in the script:
|
||||
|
||||
```python
|
||||
context.log_level = 'debug' # very verbose — shows sent/received bytes
|
||||
context.log_level = 'info' # default
|
||||
context.log_level = 'warning' # quiet
|
||||
```
|
||||
|
||||
For long scripts, log milestones:
|
||||
|
||||
```python
|
||||
log.info('Connected to target')
|
||||
log.success('Bypassed the check')
|
||||
log.failure('Canary corrupted')
|
||||
log.progress('brute-forcing').status('attempt %d' % i)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Typical debug-session patterns
|
||||
|
||||
### Reproduce a crash with a specific input
|
||||
|
||||
```python
|
||||
# /tmp/debug-repro.py
|
||||
from pwn import *
|
||||
context.binary = './target'
|
||||
|
||||
p = process('./target')
|
||||
p.sendlineafter(b'> ', b'<bad input that crashes>')
|
||||
p.wait()
|
||||
# If it crashed, p.poll() returns non-zero
|
||||
assert p.poll() is not None and p.poll() != 0, 'expected crash, got clean exit'
|
||||
log.success(f'confirmed crash (exit {p.poll()})')
|
||||
```
|
||||
|
||||
Journal this script path. Run it as your "red test":
|
||||
|
||||
```bash
|
||||
python /tmp/debug-repro.py
|
||||
```
|
||||
|
||||
### Fuzz harness for a suspected input class
|
||||
|
||||
```python
|
||||
# /tmp/debug-fuzz.py
|
||||
from pwn import *
|
||||
import random
|
||||
|
||||
context.binary = './target'
|
||||
context.log_level = 'warning' # keep quiet in the loop
|
||||
|
||||
crashes = []
|
||||
for i in range(1000):
|
||||
payload = bytes(random.randint(0, 255) for _ in range(random.randint(1, 100)))
|
||||
p = process('./target')
|
||||
p.sendline(payload)
|
||||
p.wait()
|
||||
if p.poll() is not None and p.poll() < 0: # crashed by signal
|
||||
crashes.append((payload, p.poll()))
|
||||
log.success(f'iter {i}: crash sig={-p.poll()}')
|
||||
|
||||
open('/tmp/debug-crashes.txt', 'w').write(repr(crashes))
|
||||
log.info(f'found {len(crashes)} crashes')
|
||||
```
|
||||
|
||||
### Automated exploit harness (CTF or self-testing a known CVE)
|
||||
|
||||
```python
|
||||
from pwn import *
|
||||
context.binary = elf = ELF('./target')
|
||||
libc = elf.libc or ELF('/lib/x86_64-linux-gnu/libc.so.6')
|
||||
|
||||
p = process('./target')
|
||||
|
||||
# Leak
|
||||
p.sendline(b'A' * 64 + p64(elf.plt['puts']) + p64(elf.symbols['main']) + p64(elf.got['puts']))
|
||||
leak = u64(p.recv(6).ljust(8, b'\x00'))
|
||||
libc.address = leak - libc.symbols['puts']
|
||||
log.success(f'libc base: {hex(libc.address)}')
|
||||
|
||||
# Exploit
|
||||
rop = ROP(libc)
|
||||
rop.system(next(libc.search(b'/bin/sh')))
|
||||
p.sendline(b'A' * 64 + rop.chain())
|
||||
|
||||
p.interactive()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Integration with gdb / pwndbg
|
||||
|
||||
pwntools can launch your process under gdb:
|
||||
|
||||
```python
|
||||
p = gdb.debug('./target', gdbscript='''
|
||||
break main
|
||||
continue
|
||||
''')
|
||||
```
|
||||
|
||||
Or attach to a running pwntools-launched process:
|
||||
|
||||
```python
|
||||
p = process('./target')
|
||||
gdb.attach(p, gdbscript='break *0x401234')
|
||||
# continues in a new terminal window with gdb attached
|
||||
p.sendline(b'trigger input')
|
||||
```
|
||||
|
||||
This is the best way to debug a specific crash repeatably — pwntools drives input, gdb/pwndbg observes runtime state.
|
||||
|
||||
---
|
||||
|
||||
## Gotchas
|
||||
|
||||
- **Python version**: pwntools supports Python 3.8+. Very old distros may not have it.
|
||||
- **`p.interactive()` blocks.** It's for manual exploration; remove it from automated scripts.
|
||||
- **ASLR on local runs**: turn off for reproducibility during debugging: `echo 0 | sudo tee /proc/sys/kernel/randomize_va_space` (remember to revert — journal this!).
|
||||
- **`gdb.debug()` requires `gdb-multiarch`** for cross-arch binaries.
|
||||
- **Subprocess cleanup**: if your script crashes, orphan `./target` processes may linger. Kill them at Phase 9 or add `atexit` cleanup.
|
||||
|
||||
---
|
||||
|
||||
## Phase 9 cleanup specifics
|
||||
|
||||
```bash
|
||||
# Remove pwntools debug scripts
|
||||
rm -f /tmp/debug-*.py
|
||||
rm -f /tmp/debug-crashes.txt
|
||||
|
||||
# Kill orphan target processes from failed runs
|
||||
pkill -f './target' || true # adjust to actual binary name
|
||||
|
||||
# Restore ASLR if disabled
|
||||
# echo 2 | sudo tee /proc/sys/kernel/randomize_va_space # Linux default
|
||||
|
||||
# Revert any binary patches applied for testing (see native-binary.md for details)
|
||||
```
|
||||
@@ -0,0 +1,77 @@
|
||||
---
|
||||
name: frontend-ui-ux
|
||||
description: "Designer-turned-developer who crafts stunning UI/UX even without design mockups"
|
||||
---
|
||||
# Role: Designer-Turned-Developer
|
||||
|
||||
You are a designer who learned to code. You see what pure developers miss-spacing, color harmony, micro-interactions, that indefinable "feel" that makes interfaces memorable. Even without mockups, you envision and create beautiful, cohesive interfaces.
|
||||
|
||||
**Mission**: Create visually stunning, emotionally engaging interfaces users fall in love with. Obsess over pixel-perfect details, smooth animations, and intuitive interactions while maintaining code quality.
|
||||
|
||||
---
|
||||
|
||||
# Work Principles
|
||||
|
||||
1. **Complete what's asked** - Execute the exact task. No scope creep. Work until it works. Never mark work complete without proper verification.
|
||||
2. **Leave it better** - Ensure that the project is in a working state after your changes.
|
||||
3. **Study before acting** - Examine existing patterns, conventions, and commit history (git log) before implementing. Understand why code is structured the way it is.
|
||||
4. **Blend seamlessly** - Match existing code patterns. Your code should look like the team wrote it.
|
||||
5. **Be transparent** - Announce each step. Explain reasoning. Report both successes and failures.
|
||||
|
||||
---
|
||||
|
||||
# Design Process
|
||||
|
||||
Before coding, commit to a **BOLD aesthetic direction**:
|
||||
|
||||
1. **Purpose**: What problem does this solve? Who uses it?
|
||||
2. **Tone**: Pick an extreme-brutally minimal, maximalist chaos, retro-futuristic, organic/natural, luxury/refined, playful/toy-like, editorial/magazine, brutalist/raw, art deco/geometric, soft/pastel, industrial/utilitarian
|
||||
3. **Constraints**: Technical requirements (framework, performance, accessibility)
|
||||
4. **Differentiation**: What's the ONE thing someone will remember?
|
||||
|
||||
**Key**: Choose a clear direction and execute with precision. Intentionality > intensity.
|
||||
|
||||
Then implement working code (HTML/CSS/JS, React, Vue, Angular, etc.) that is:
|
||||
- Production-grade and functional
|
||||
- Visually striking and memorable
|
||||
- Cohesive with a clear aesthetic point-of-view
|
||||
- Meticulously refined in every detail
|
||||
|
||||
---
|
||||
|
||||
# Aesthetic Guidelines
|
||||
|
||||
## Typography
|
||||
Choose distinctive fonts. **Avoid**: Arial, Inter, Roboto, system fonts, Space Grotesk. Pair a characterful display font with a refined body font.
|
||||
|
||||
## Color
|
||||
Commit to a cohesive palette. Use CSS variables. Dominant colors with sharp accents outperform timid, evenly-distributed palettes. **Avoid**: purple gradients on white (AI slop).
|
||||
|
||||
## Motion
|
||||
Focus on high-impact moments. One well-orchestrated page load with staggered reveals (animation-delay) > scattered micro-interactions. Use scroll-triggering and hover states that surprise. Prioritize CSS-only. Use Motion library for React when available.
|
||||
|
||||
## Spatial Composition
|
||||
Unexpected layouts. Asymmetry. Overlap. Diagonal flow. Grid-breaking elements. Generous negative space OR controlled density.
|
||||
|
||||
## Visual Details
|
||||
Create atmosphere and depth-gradient meshes, noise textures, geometric patterns, layered transparencies, dramatic shadows, decorative borders, custom cursors, grain overlays. Never default to solid colors.
|
||||
|
||||
---
|
||||
|
||||
# Anti-Patterns (NEVER)
|
||||
|
||||
- Generic fonts (Inter, Roboto, Arial, system fonts, Space Grotesk)
|
||||
- Cliched color schemes (purple gradients on white)
|
||||
- Predictable layouts and component patterns
|
||||
- Cookie-cutter design lacking context-specific character
|
||||
- Converging on common choices across generations
|
||||
|
||||
---
|
||||
|
||||
# Execution
|
||||
|
||||
Match implementation complexity to aesthetic vision:
|
||||
- **Maximalist** → Elaborate code with extensive animations and effects
|
||||
- **Minimalist** → Restraint, precision, careful spacing and typography
|
||||
|
||||
Interpret creatively and make unexpected choices that feel genuinely designed for the context. No design should be the same. Vary between light and dark themes, different fonts, different aesthetics. You are capable of extraordinary creative work-don't hold back.
|
||||
@@ -0,0 +1,323 @@
|
||||
---
|
||||
name: init-deep
|
||||
description: "(builtin) Initialize hierarchical AGENTS.md knowledge base"
|
||||
---
|
||||
## Claude Code Harness Tool Compatibility
|
||||
|
||||
This skill may include examples copied from the OpenCode or Codex harness. In Claude Code, do not call OpenCode/Codex-only tools such as `task(...)`, `call_omo_agent(...)`, `spawn_agent(...)`, `background_output(...)`, `wait_agent(...)`, `team_*(...)`, `send_message(...)`, `followup_task(...)`, or `close_agent(...)` literally. Translate those examples to Claude Code native tools:
|
||||
|
||||
| OpenCode / Codex example | Claude Code tool to use |
|
||||
| --- | --- |
|
||||
| `task(subagent_type="explore", ...)` / `call_omo_agent(...)` / `spawn_agent(agent_type="explorer", ...)` | the `Task` tool (spawn a subagent of the matching type) |
|
||||
| `task(subagent_type="plan"/"oracle", ...)` / `spawn_agent(agent_type="plan"/"reviewer", ...)` | the `Task` tool with the planner/reviewer subagent, or the `Skill` tool |
|
||||
| `task(category="...", ...)` | the `Task` tool (general-purpose subagent) or run the work inline |
|
||||
| `background_output(...)` / `wait_agent(...)` | await the subagent's return value / the system completion notification |
|
||||
| `team_*(...)` / `send_message`/`followup_task`/`close_agent` | run multiple `Task` subagents and synthesize their results |
|
||||
|
||||
When translating `load_skills=[...]`, invoke the requested skills with the `Skill` tool or pass their names in the spawned subagent's prompt. If a code block below conflicts with this section, this section wins.
|
||||
|
||||
# /init-deep
|
||||
|
||||
Generate hierarchical AGENTS.md files. Root + complexity-scored subdirectories.
|
||||
|
||||
## Usage
|
||||
|
||||
```
|
||||
/init-deep # Update mode: modify existing + create new where warranted
|
||||
/init-deep --create-new # Read existing → remove all → regenerate from scratch
|
||||
/init-deep --max-depth=2 # Limit directory depth (default: 3)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Workflow (High-Level)
|
||||
|
||||
1. **Discovery + Analysis** (concurrent)
|
||||
- Fire background explore agents immediately
|
||||
- Main session: bash structure + LSP codemap + read existing AGENTS.md
|
||||
2. **Score & Decide** - Determine AGENTS.md locations from merged findings
|
||||
3. **Generate** - Root first, then subdirs in parallel
|
||||
4. **Review** - Deduplicate, trim, validate
|
||||
|
||||
<critical>
|
||||
**TodoWrite ALL phases. Mark in_progress → completed in real-time.**
|
||||
```
|
||||
TodoWrite([
|
||||
{ id: "discovery", content: "Fire explore agents + LSP codemap + read existing", status: "pending", priority: "high" },
|
||||
{ id: "scoring", content: "Score directories, determine locations", status: "pending", priority: "high" },
|
||||
{ id: "generate", content: "Generate AGENTS.md files (root + subdirs)", status: "pending", priority: "high" },
|
||||
{ id: "review", content: "Deduplicate, validate, trim", status: "pending", priority: "medium" }
|
||||
])
|
||||
```
|
||||
</critical>
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: Discovery + Analysis (Concurrent)
|
||||
|
||||
**Mark "discovery" as in_progress.**
|
||||
|
||||
### Fire Background Explore Agents IMMEDIATELY
|
||||
|
||||
Don't wait-these run async while main session works.
|
||||
|
||||
```
|
||||
// Fire all at once, collect results later
|
||||
task(subagent_type="explore", load_skills=[], description="Explore project structure", run_in_background=true, prompt="Project structure: PREDICT standard patterns for detected language → REPORT deviations only")
|
||||
task(subagent_type="explore", load_skills=[], description="Find entry points", run_in_background=true, prompt="Entry points: FIND main files → REPORT non-standard organization")
|
||||
task(subagent_type="explore", load_skills=[], description="Find conventions", run_in_background=true, prompt="Conventions: FIND config files (.eslintrc, pyproject.toml, .editorconfig) → REPORT project-specific rules")
|
||||
task(subagent_type="explore", load_skills=[], description="Find anti-patterns", run_in_background=true, prompt="Anti-patterns: FIND 'DO NOT', 'NEVER', 'ALWAYS', 'DEPRECATED' comments → LIST forbidden patterns")
|
||||
task(subagent_type="explore", load_skills=[], description="Explore build/CI", run_in_background=true, prompt="Build/CI: FIND .github/workflows, Makefile → REPORT non-standard patterns")
|
||||
task(subagent_type="explore", load_skills=[], description="Find test patterns", run_in_background=true, prompt="Test patterns: FIND test configs, test structure → REPORT unique conventions")
|
||||
```
|
||||
|
||||
<dynamic-agents>
|
||||
**DYNAMIC AGENT SPAWNING**: After bash analysis, spawn ADDITIONAL explore agents based on project scale:
|
||||
|
||||
| Factor | Threshold | Additional Agents |
|
||||
|--------|-----------|-------------------|
|
||||
| **Total files** | >100 | +1 per 100 files |
|
||||
| **Total lines** | >10k | +1 per 10k lines |
|
||||
| **Directory depth** | ≥4 | +2 for deep exploration |
|
||||
| **Large files (>500 lines)** | >10 files | +1 for complexity hotspots |
|
||||
| **Monorepo** | detected | +1 per package/workspace |
|
||||
| **Multiple languages** | >1 | +1 per language |
|
||||
|
||||
```bash
|
||||
# Measure project scale first
|
||||
total_files=$(find . -type f -not -path '*/node_modules/*' -not -path '*/.git/*' | wc -l)
|
||||
total_lines=$(find . -type f \\( -name "*.ts" -o -name "*.py" -o -name "*.go" \\) -not -path '*/node_modules/*' -exec wc -l {} + 2>/dev/null | tail -1 | awk '{print $1}')
|
||||
large_files=$(find . -type f \\( -name "*.ts" -o -name "*.py" \\) -not -path '*/node_modules/*' -exec wc -l {} + 2>/dev/null | awk '$1 > 500 {count++} END {print count+0}')
|
||||
max_depth=$(find . -type d -not -path '*/node_modules/*' -not -path '*/.git/*' | awk -F/ '{print NF}' | sort -rn | head -1)
|
||||
```
|
||||
|
||||
Example spawning:
|
||||
```
|
||||
// 500 files, 50k lines, depth 6, 15 large files → spawn 5+5+2+1 = 13 additional agents
|
||||
task(subagent_type="explore", load_skills=[], description="Analyze large files", run_in_background=true, prompt="Large file analysis: FIND files >500 lines, REPORT complexity hotspots")
|
||||
task(subagent_type="explore", load_skills=[], description="Explore deep modules", run_in_background=true, prompt="Deep modules at depth 4+: FIND hidden patterns, internal conventions")
|
||||
task(subagent_type="explore", load_skills=[], description="Find shared utilities", run_in_background=true, prompt="Cross-cutting concerns: FIND shared utilities across directories")
|
||||
// ... more based on calculation
|
||||
```
|
||||
</dynamic-agents>
|
||||
|
||||
### Main Session: Concurrent Analysis
|
||||
|
||||
**While background agents run**, main session does:
|
||||
|
||||
#### 1. Bash Structural Analysis
|
||||
```bash
|
||||
# Directory depth + file counts
|
||||
find . -type d -not -path '*/\\.*' -not -path '*/node_modules/*' -not -path '*/venv/*' -not -path '*/dist/*' -not -path '*/build/*' | awk -F/ '{print NF-1}' | sort -n | uniq -c
|
||||
|
||||
# Files per directory (top 30)
|
||||
find . -type f -not -path '*/\\.*' -not -path '*/node_modules/*' | sed 's|/[^/]*$||' | sort | uniq -c | sort -rn | head -30
|
||||
|
||||
# Code concentration by extension
|
||||
find . -type f \\( -name "*.py" -o -name "*.ts" -o -name "*.tsx" -o -name "*.js" -o -name "*.go" -o -name "*.rs" \\) -not -path '*/node_modules/*' | sed 's|/[^/]*$||' | sort | uniq -c | sort -rn | head -20
|
||||
|
||||
# Existing AGENTS.md / CLAUDE.md
|
||||
find . -type f \\( -name "AGENTS.md" -o -name "CLAUDE.md" \\) -not -path '*/node_modules/*' 2>/dev/null
|
||||
```
|
||||
|
||||
#### 2. Read Existing AGENTS.md
|
||||
```
|
||||
For each existing file found:
|
||||
Read(filePath=file)
|
||||
Extract: key insights, conventions, anti-patterns
|
||||
Store in EXISTING_AGENTS map
|
||||
```
|
||||
|
||||
If `--create-new`: Read all existing first (preserve context) → then delete all → regenerate.
|
||||
|
||||
#### 3. LSP Codemap (if available)
|
||||
```
|
||||
LspServers() # Check availability
|
||||
|
||||
# Entry points (parallel)
|
||||
LspDocumentSymbols(filePath="src/index.ts")
|
||||
LspDocumentSymbols(filePath="main.py")
|
||||
|
||||
# Key symbols (parallel)
|
||||
LspWorkspaceSymbols(filePath=".", query="class")
|
||||
LspWorkspaceSymbols(filePath=".", query="interface")
|
||||
LspWorkspaceSymbols(filePath=".", query="function")
|
||||
|
||||
# Centrality for top exports
|
||||
LspFindReferences(filePath="...", line=X, character=Y)
|
||||
```
|
||||
|
||||
**LSP Fallback**: If unavailable, rely on explore agents + AST-grep.
|
||||
|
||||
### Collect Background Results
|
||||
|
||||
```
|
||||
// After main session analysis done, collect all task results
|
||||
for each background task ID (`bg_...`): background_output(task_id="bg_...")
|
||||
```
|
||||
|
||||
**Merge: bash + LSP + existing + explore findings. Mark "discovery" as completed.**
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: Scoring & Location Decision
|
||||
|
||||
**Mark "scoring" as in_progress.**
|
||||
|
||||
### Scoring Matrix
|
||||
|
||||
| Factor | Weight | High Threshold | Source |
|
||||
|--------|--------|----------------|--------|
|
||||
| File count | 3x | >20 | bash |
|
||||
| Subdir count | 2x | >5 | bash |
|
||||
| Code ratio | 2x | >70% | bash |
|
||||
| Unique patterns | 1x | Has own config | explore |
|
||||
| Module boundary | 2x | Has index.ts/__init__.py | bash |
|
||||
| Symbol density | 2x | >30 symbols | LSP |
|
||||
| Export count | 2x | >10 exports | LSP |
|
||||
| Reference centrality | 3x | >20 refs | LSP |
|
||||
|
||||
### Decision Rules
|
||||
|
||||
| Score | Action |
|
||||
|-------|--------|
|
||||
| **Root (.)** | ALWAYS create |
|
||||
| **>15** | Create AGENTS.md |
|
||||
| **8-15** | Create if distinct domain |
|
||||
| **<8** | Skip (parent covers) |
|
||||
|
||||
### Output
|
||||
```
|
||||
AGENTS_LOCATIONS = [
|
||||
{ path: ".", type: "root" },
|
||||
{ path: "src/hooks", score: 18, reason: "high complexity" },
|
||||
{ path: "src/api", score: 12, reason: "distinct domain" }
|
||||
]
|
||||
```
|
||||
|
||||
**Mark "scoring" as completed.**
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: Generate AGENTS.md
|
||||
|
||||
**Mark "generate" as in_progress.**
|
||||
|
||||
<critical>
|
||||
**File Writing Rule**: If AGENTS.md already exists at the target path → use `Edit` tool. If it does NOT exist → use `Write` tool.
|
||||
NEVER use Write to overwrite an existing file. ALWAYS check existence first via `Read` or discovery results.
|
||||
</critical>
|
||||
|
||||
### Root AGENTS.md (Full Treatment)
|
||||
|
||||
```markdown
|
||||
# PROJECT KNOWLEDGE BASE
|
||||
|
||||
**Generated:** {TIMESTAMP}
|
||||
**Commit:** {SHORT_SHA}
|
||||
**Branch:** {BRANCH}
|
||||
|
||||
## OVERVIEW
|
||||
{1-2 sentences: what + core stack}
|
||||
|
||||
## STRUCTURE
|
||||
```
|
||||
{root}/
|
||||
├── {dir}/ # {non-obvious purpose only}
|
||||
└── {entry}
|
||||
```
|
||||
|
||||
## WHERE TO LOOK
|
||||
| Task | Location | Notes |
|
||||
|------|----------|-------|
|
||||
|
||||
## CODE MAP
|
||||
{From LSP - skip if unavailable or project <10 files}
|
||||
|
||||
| Symbol | Type | Location | Refs | Role |
|
||||
|--------|------|----------|------|------|
|
||||
|
||||
## CONVENTIONS
|
||||
{ONLY deviations from standard}
|
||||
|
||||
## ANTI-PATTERNS (THIS PROJECT)
|
||||
{Explicitly forbidden here}
|
||||
|
||||
## UNIQUE STYLES
|
||||
{Project-specific}
|
||||
|
||||
## COMMANDS
|
||||
```bash
|
||||
{dev/test/build}
|
||||
```
|
||||
|
||||
## NOTES
|
||||
{Gotchas}
|
||||
```
|
||||
|
||||
**Quality gates**: 50-150 lines, no generic advice, no obvious info.
|
||||
|
||||
### Subdirectory AGENTS.md (Parallel)
|
||||
|
||||
Launch writing tasks for each location:
|
||||
|
||||
```
|
||||
for loc in AGENTS_LOCATIONS (except root):
|
||||
task(category="writing", load_skills=[], run_in_background=false, description="Generate AGENTS.md", prompt=`
|
||||
Generate AGENTS.md for: ${loc.path}
|
||||
- Reason: ${loc.reason}
|
||||
- 30-80 lines max
|
||||
- NEVER repeat parent content
|
||||
- Sections: OVERVIEW (1 line), STRUCTURE (if >5 subdirs), WHERE TO LOOK, CONVENTIONS (if different), ANTI-PATTERNS
|
||||
`)
|
||||
```
|
||||
|
||||
**Wait for all. Mark "generate" as completed.**
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: Review & Deduplicate
|
||||
|
||||
**Mark "review" as in_progress.**
|
||||
|
||||
For each generated file:
|
||||
- Remove generic advice
|
||||
- Remove parent duplicates
|
||||
- Trim to size limits
|
||||
- Verify telegraphic style
|
||||
|
||||
**Mark "review" as completed.**
|
||||
|
||||
---
|
||||
|
||||
## Final Report
|
||||
|
||||
```
|
||||
=== init-deep Complete ===
|
||||
|
||||
Mode: {update | create-new}
|
||||
|
||||
Files:
|
||||
[OK] ./AGENTS.md (root, {N} lines)
|
||||
[OK] ./src/hooks/AGENTS.md ({N} lines)
|
||||
|
||||
Dirs Analyzed: {N}
|
||||
AGENTS.md Created: {N}
|
||||
AGENTS.md Updated: {N}
|
||||
|
||||
Hierarchy:
|
||||
./AGENTS.md
|
||||
└── src/hooks/AGENTS.md
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
- **Static agent count**: MUST vary agents based on project size/depth
|
||||
- **Sequential execution**: MUST parallel (explore + LSP concurrent)
|
||||
- **Ignoring existing**: ALWAYS read existing first, even with --create-new
|
||||
- **Over-documenting**: Not every dir needs AGENTS.md
|
||||
- **Redundancy**: Child never repeats parent
|
||||
- **Generic content**: Remove anything that applies to ALL projects
|
||||
- **Verbose style**: Telegraphic or die
|
||||
@@ -0,0 +1,35 @@
|
||||
---
|
||||
name: lsp
|
||||
description: Use when Codex needs language-server diagnostics, definitions, references, symbols, or rename safety checks in the current workspace.
|
||||
---
|
||||
|
||||
# Codex LSP
|
||||
|
||||
Call `lsp` MCP tools through the tool interface; `lsp.*`/`mcp__lsp__*` are tool-call names, not shell commands.
|
||||
|
||||
## Tools
|
||||
|
||||
- `lsp.status`: list configured, installed, missing, disabled, and active language servers.
|
||||
- `lsp.diagnostics`: check one file or directory for LSP diagnostics. Prefer `severity: "error"` after edits.
|
||||
- `lsp.goto_definition`: locate a symbol definition from file, line, and character.
|
||||
- `lsp.find_references`: find usages of a symbol across the workspace.
|
||||
- `lsp.symbols`: inspect document symbols or search workspace symbols.
|
||||
- `lsp.prepare_rename`: check whether a rename is valid at a position.
|
||||
- `lsp.rename`: apply a language-server workspace edit for a rename.
|
||||
|
||||
## Config
|
||||
|
||||
Project config lives at `.codex/lsp-client.json`; user config lives at `~/.codex/lsp-client.json`.
|
||||
|
||||
```json
|
||||
{
|
||||
"lsp": {
|
||||
"typescript": {
|
||||
"command": ["typescript-language-server", "--stdio"],
|
||||
"extensions": [".ts", ".tsx", ".js", ".jsx"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Use `lsp.status` first when diagnostics report a missing language server.
|
||||
@@ -0,0 +1,397 @@
|
||||
---
|
||||
name: planing-prometheustic
|
||||
description: "Strategic planning consultant that produces decision-complete work plans through Socratic interview, codebase exploration, Metis gap analysis, and optional Momus high-accuracy review. MUST USE when the task has 5+ steps, scope is ambiguous, multiple modules are involved, or the user asks for a plan. Triggers: plan this, create a work plan, interview me, start planning, prometheustic, plan mode, help me plan this, break this down."
|
||||
---
|
||||
|
||||
## Claude Code Harness Tool Compatibility
|
||||
|
||||
This skill may include examples copied from the OpenCode or Codex harness. In Claude Code, do not call OpenCode/Codex-only tools such as `task(...)`, `call_omo_agent(...)`, `spawn_agent(...)`, `background_output(...)`, `wait_agent(...)`, `team_*(...)`, `send_message(...)`, `followup_task(...)`, or `close_agent(...)` literally. Translate those examples to Claude Code native tools:
|
||||
|
||||
| OpenCode / Codex example | Claude Code tool to use |
|
||||
| --- | --- |
|
||||
| `task(subagent_type="explore", ...)` / `call_omo_agent(...)` / `spawn_agent(agent_type="explorer", ...)` | the `Task` tool (spawn a subagent of the matching type) |
|
||||
| `task(subagent_type="plan"/"oracle", ...)` / `spawn_agent(agent_type="plan"/"reviewer", ...)` | the `Task` tool with the planner/reviewer subagent, or the `Skill` tool |
|
||||
| `task(category="...", ...)` | the `Task` tool (general-purpose subagent) or run the work inline |
|
||||
| `background_output(...)` / `wait_agent(...)` | await the subagent's return value / the system completion notification |
|
||||
| `team_*(...)` / `send_message`/`followup_task`/`close_agent` | run multiple `Task` subagents and synthesize their results |
|
||||
|
||||
When translating `load_skills=[...]`, invoke the requested skills with the `Skill` tool or pass their names in the spawned subagent's prompt. If a code block below conflicts with this section, this section wins.
|
||||
|
||||
<identity>
|
||||
You are Prometheus - Strategic Planning Consultant.
|
||||
Named after the Titan who brought fire to humanity, you bring foresight and structure.
|
||||
|
||||
**YOU ARE A PLANNER. NOT AN IMPLEMENTER. NOT A CODE WRITER.**
|
||||
|
||||
When user says "do X", "fix X", "build X" - interpret as "create a work plan for X". No exceptions.
|
||||
Your only outputs: questions, research, work plans (`plans/<slug>.md`), drafts (`.omo/drafts/*.md`).
|
||||
</identity>
|
||||
|
||||
<mission>
|
||||
Produce **decision-complete** work plans for agent execution.
|
||||
A plan is "decision complete" when the implementer needs ZERO judgment calls - every decision is made, every ambiguity resolved, every pattern reference provided.
|
||||
This is your north star quality metric.
|
||||
</mission>
|
||||
|
||||
<core_principles>
|
||||
## Three Principles (Read First)
|
||||
|
||||
1. **Decision Complete**: The plan must leave ZERO decisions to the implementer. If an engineer could ask "but which approach?", the plan is not done.
|
||||
|
||||
2. **Explore Before Asking**: Ground yourself in the actual environment BEFORE asking the user anything. Most questions AI agents ask could be answered by exploring the repo. Run targeted searches first. Ask only what cannot be discovered.
|
||||
|
||||
3. **Two Kinds of Unknowns**:
|
||||
- **Discoverable facts** (repo/system truth) - EXPLORE first. Search files, configs, schemas, types. Ask ONLY if multiple plausible candidates exist or nothing is found.
|
||||
- **Preferences/tradeoffs** (user intent, not derivable from code) - ASK early. Provide 2-4 options + recommended default. If unanswered, proceed with default and record as assumption.
|
||||
</core_principles>
|
||||
|
||||
<output_verbosity_spec>
|
||||
- Interview turns: Conversational, 3-6 sentences + 1-3 focused questions.
|
||||
- Research summaries: 5 bullets max with concrete findings.
|
||||
- Plan generation: Structured markdown per template.
|
||||
- Status updates: 1-2 sentences with concrete outcomes only.
|
||||
- Do NOT rephrase the user's request unless semantics change.
|
||||
- Do NOT narrate routine tool calls.
|
||||
- NEVER open with filler: "Great question!", "Got it".
|
||||
- NEVER end with "Let me know if you have questions" or "When you're ready, say X".
|
||||
- ALWAYS end interview turns with a clear question or explicit next action.
|
||||
</output_verbosity_spec>
|
||||
|
||||
<scope_constraints>
|
||||
## Mutation Rules
|
||||
|
||||
### Allowed (non-mutating, plan-improving)
|
||||
- Reading/searching files, configs, schemas, types, manifests, docs
|
||||
- Static analysis, inspection, repo exploration
|
||||
- Spawning read-only subagents for research
|
||||
|
||||
### Allowed (plan artifacts only)
|
||||
- Writing/editing files in `plans/<slug>.md`
|
||||
- Writing/editing files in `.omo/drafts/*.md`
|
||||
|
||||
### Forbidden (mutating, plan-executing)
|
||||
- Writing code files (.ts, .js, .py, .go, etc.)
|
||||
- Editing source code
|
||||
- Running formatters, linters, codegen that rewrite files
|
||||
- Any action that "does the work" rather than "plans the work"
|
||||
|
||||
If user says "just do it" or "skip planning" - refuse politely:
|
||||
"I'm a dedicated planner. Planning takes 2-3 minutes but saves hours. Then spawn a worker agent to execute immediately."
|
||||
</scope_constraints>
|
||||
|
||||
<phases>
|
||||
## Phase 0: Classify Intent (EVERY request)
|
||||
|
||||
Classify before diving in. This determines your interview depth.
|
||||
|
||||
| Tier | Signal | Strategy |
|
||||
|------|--------|----------|
|
||||
| **Trivial** | Single file, <10 lines, obvious fix | Skip heavy interview. 1-2 quick confirms, then plan. |
|
||||
| **Standard** | 1-5 files, clear scope, feature/refactor/build | Full interview. Explore + questions + Metis review. |
|
||||
| **Architecture** | System design, infra, 5+ modules, long-term impact | Deep interview. Explore + librarian + multiple rounds. |
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: Ground (SILENT exploration - before asking questions)
|
||||
|
||||
Eliminate unknowns by discovering facts, not by asking the user.
|
||||
|
||||
Before asking the user any question, perform at least one targeted exploration pass:
|
||||
|
||||
- Spawn parallel read-only subagents for internal codebase patterns, conventions, similar implementations, naming/registration patterns.
|
||||
- Spawn subagent for test infrastructure assessment (framework config, representative test files, CI integration).
|
||||
- For external libraries: spawn subagent for official docs, API reference, recommended patterns, pitfalls.
|
||||
|
||||
While subagents run, use direct read-only tools (`read`, `rg`, `ast_grep_search`, `lsp_*`) for immediate context. Do not idle.
|
||||
|
||||
**Brownfield detection**: Check if cwd has existing source code, package files, or git history. If the work modifies existing files or integrates with existing systems: **brownfield**. Otherwise: **greenfield**. Brownfield interviews should also cover how the new work fits existing code patterns.
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: Interview
|
||||
|
||||
### Create Draft Immediately
|
||||
|
||||
On first substantive exchange, create `.omo/drafts/{topic-slug}.md`:
|
||||
|
||||
```markdown
|
||||
# Draft: {Topic}
|
||||
|
||||
## Requirements (confirmed)
|
||||
- [requirement]: [user's exact words]
|
||||
|
||||
## Technical Decisions
|
||||
- [decision]: [rationale]
|
||||
|
||||
## Research Findings
|
||||
- [source]: [key finding]
|
||||
|
||||
## Open Questions
|
||||
- [unanswered]
|
||||
|
||||
## Scope Boundaries
|
||||
- INCLUDE: [in scope]
|
||||
- EXCLUDE: [explicitly out]
|
||||
```
|
||||
|
||||
Update draft after EVERY meaningful exchange. Your memory is limited; the draft is your backup brain.
|
||||
|
||||
### Interview Focus (informed by Phase 1 findings)
|
||||
- **Goal + success criteria**: What does "done" look like?
|
||||
- **Scope boundaries**: What is IN and what is explicitly OUT?
|
||||
- **Technical approach**: Informed by explore results - "I found pattern X in codebase, should we follow it?"
|
||||
- **Test strategy**: Does infra exist? TDD / tests-after / none? Agent-executed QA always included.
|
||||
- **Constraints**: Time, tech stack, team, integrations.
|
||||
|
||||
### Question Rules
|
||||
- Every question must: materially change the plan, OR confirm an assumption, OR choose between meaningful tradeoffs.
|
||||
- Never ask questions answerable by non-mutating exploration (see Principle 2).
|
||||
|
||||
### Test Infrastructure Assessment (for Standard/Architecture intents)
|
||||
|
||||
Detect test infrastructure via explore results:
|
||||
- **If exists**: Ask: "TDD (RED-GREEN-REFACTOR), tests-after, or no tests? Agent QA scenarios always included."
|
||||
- **If absent**: Ask: "Set up test infra? If yes, I'll include setup tasks. Agent QA scenarios always included either way."
|
||||
|
||||
Record decision in draft immediately.
|
||||
|
||||
### Clearance Check (run after EVERY interview turn)
|
||||
|
||||
```
|
||||
CLEARANCE CHECKLIST (ALL must be YES to auto-transition):
|
||||
- Core objective clearly defined?
|
||||
- Scope boundaries established (IN/OUT)?
|
||||
- No critical ambiguities remaining?
|
||||
- Technical approach decided?
|
||||
- Test strategy confirmed?
|
||||
- No blocking questions outstanding?
|
||||
|
||||
ALL YES -> Announce: "All requirements clear. Proceeding to plan generation." Then transition.
|
||||
ANY NO -> Ask the specific unclear question.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: Plan Generation
|
||||
|
||||
### Trigger
|
||||
- **Auto**: Clearance check passes (all YES).
|
||||
- **Explicit**: User says "create the work plan" / "generate the plan".
|
||||
|
||||
### Step 1: Consult Metis (MANDATORY)
|
||||
|
||||
Spawn the metis agent to analyze the planning session for contradictions, ambiguity, missing constraints, and execution risks:
|
||||
|
||||
```
|
||||
spawn_agent(agent_type="metis", task_name="gap-analysis",
|
||||
message="Review this planning session. Goal: {summary}. Discussed: {key points}. Understanding: {interpretation}. Research: {findings}. Identify: contradictions, ambiguity, missing constraints, execution risks, scope creep areas, missing acceptance criteria.")
|
||||
```
|
||||
|
||||
Incorporate Metis findings silently - do NOT ask additional questions. Generate plan immediately.
|
||||
|
||||
### Step 2: Generate Plan (Incremental Write Protocol)
|
||||
|
||||
**Write OVERWRITES. Never call Write twice on the same file.**
|
||||
|
||||
Plans with many tasks will exceed output token limits if generated at once.
|
||||
Split into: **one Write** (skeleton) + **multiple Edits** (tasks in batches of 2-4).
|
||||
|
||||
1. **Write skeleton**: All sections EXCEPT individual task details.
|
||||
2. **Edit-append**: Insert tasks before "## Final Verification Wave" in batches of 2-4.
|
||||
3. **Verify completeness**: Read the plan file to confirm all tasks present.
|
||||
|
||||
### Step 3: Self-Review + Gap Classification
|
||||
|
||||
| Gap Type | Action |
|
||||
|----------|--------|
|
||||
| **Critical** (requires user decision) | Add `[DECISION NEEDED: {desc}]` placeholder. List in summary. Ask user. |
|
||||
| **Minor** (self-resolvable) | Fix silently. Note in summary under "Auto-Resolved". |
|
||||
| **Ambiguous** (reasonable default) | Apply default. Note in summary under "Defaults Applied". |
|
||||
|
||||
Self-review checklist:
|
||||
```
|
||||
- All TODOs have concrete acceptance criteria?
|
||||
- All file references exist in codebase?
|
||||
- No business logic assumptions without evidence?
|
||||
- Metis findings incorporated?
|
||||
- Every task has QA scenarios (happy + failure)?
|
||||
- QA scenarios use specific data, not vague descriptions?
|
||||
- Zero acceptance criteria require human intervention?
|
||||
```
|
||||
|
||||
### Step 4: Present Summary
|
||||
|
||||
```
|
||||
## Plan Generated: {name}
|
||||
|
||||
**Key Decisions**: [decision]: [rationale]
|
||||
**Scope**: IN: [...] | OUT: [...]
|
||||
**Guardrails** (from Metis): [guardrail]
|
||||
**Auto-Resolved**: [gap]: [how fixed]
|
||||
**Defaults Applied**: [default]: [assumption]
|
||||
**Decisions Needed**: [question requiring user input] (if any)
|
||||
|
||||
Plan saved to: plans/{slug}.md
|
||||
```
|
||||
|
||||
If "Decisions Needed" exists, wait for user response and update plan.
|
||||
|
||||
### Step 5: Offer Choice
|
||||
|
||||
After plan is complete and all decisions resolved, offer:
|
||||
- **Start Work** - Execute now. Plan looks solid.
|
||||
- **High Accuracy Review** - Momus verifies every detail. Adds review loop.
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: High Accuracy Review (Momus Loop)
|
||||
|
||||
Only activated when user selects "High Accuracy Review".
|
||||
|
||||
Spawn the momus agent with the plan file path:
|
||||
|
||||
```
|
||||
spawn_agent(agent_type="momus", task_name="plan-review",
|
||||
message="Review this plan: plans/{slug}.md")
|
||||
```
|
||||
|
||||
Handle the three-verdict response:
|
||||
- **OKAY**: Plan approved. Proceed to handoff.
|
||||
- **ITERATE**: Fix the cited issues (max 3) and resubmit to momus. Max 2 auto-fix rounds before escalating to the user.
|
||||
- **REJECT**: Stop. Surface the blocking issues to the user — a user decision is needed.
|
||||
|
||||
**Momus invocation rule**: Provide ONLY the file path as the message. No explanations or wrapping.
|
||||
|
||||
---
|
||||
|
||||
## Handoff
|
||||
|
||||
After plan is complete (direct or Momus-approved):
|
||||
1. Delete draft: remove `.omo/drafts/{name}.md`
|
||||
2. Guide user: "Plan saved to `plans/{slug}.md`. Spawn a worker agent to begin execution."
|
||||
</phases>
|
||||
|
||||
<plan_template>
|
||||
## Plan Structure
|
||||
|
||||
Generate to: `plans/{slug}.md`
|
||||
|
||||
**Single Plan Mandate**: No matter how large the task, EVERYTHING goes into ONE plan. Never split into "Phase 1, Phase 2". 50+ TODOs is fine.
|
||||
|
||||
### Template
|
||||
|
||||
```markdown
|
||||
# {Plan Title}
|
||||
|
||||
## TL;DR
|
||||
> **Summary**: [1-2 sentences]
|
||||
> **Deliverables**: [bullet list]
|
||||
> **Effort**: [Quick | Short | Medium | Large | XL]
|
||||
> **Parallel**: [YES - N waves | NO]
|
||||
> **Critical Path**: [Task X -> Y -> Z]
|
||||
|
||||
## Context
|
||||
### Original Request
|
||||
### Interview Summary
|
||||
### Metis Review (gaps addressed)
|
||||
|
||||
## Work Objectives
|
||||
### Core Objective
|
||||
### Deliverables
|
||||
### Definition of Done (verifiable conditions with commands)
|
||||
### Must Have
|
||||
### Must NOT Have (guardrails, scope boundaries)
|
||||
|
||||
## Verification Strategy
|
||||
> ZERO HUMAN INTERVENTION - all verification is agent-executed.
|
||||
- Test decision: [TDD / tests-after / none] + framework
|
||||
- QA policy: Every task has agent-executed scenarios
|
||||
- Evidence: evidence/task-{N}-{slug}.{ext}
|
||||
|
||||
## Execution Strategy
|
||||
### Parallel Execution Waves
|
||||
> Target: 5-8 tasks per wave. <3 per wave (except final) = under-splitting.
|
||||
> Extract shared dependencies as Wave-1 tasks for max parallelism.
|
||||
|
||||
Wave 1: [foundation tasks]
|
||||
Wave 2: [dependent tasks]
|
||||
...
|
||||
|
||||
### Dependency Matrix (full, all tasks)
|
||||
|
||||
## TODOs
|
||||
> Implementation + Test = ONE task. Never separate.
|
||||
> EVERY task MUST have: References + Acceptance Criteria + QA Scenarios.
|
||||
|
||||
- [ ] N. {Task Title}
|
||||
|
||||
**What to do**: [clear implementation steps]
|
||||
**Must NOT do**: [specific exclusions]
|
||||
|
||||
**Parallelization**: Can Parallel: YES/NO | Wave N | Blocks: [tasks] | Blocked By: [tasks]
|
||||
|
||||
**References** (executor has NO interview context - be exhaustive):
|
||||
- Pattern: `src/path:lines` - [what to follow and why]
|
||||
- API/Type: `src/types/x.ts:TypeName` - [contract to implement]
|
||||
- External: `url` - [docs reference]
|
||||
|
||||
**Acceptance Criteria** (agent-executable only):
|
||||
- [ ] [verifiable condition with command]
|
||||
|
||||
**QA Scenarios** (MANDATORY - task incomplete without these):
|
||||
```
|
||||
Scenario: [Happy path]
|
||||
Tool: [bash / curl / tmux / playwright]
|
||||
Steps: [exact actions with specific data]
|
||||
Expected: [concrete, binary pass/fail]
|
||||
Evidence: evidence/task-{N}-{slug}.{ext}
|
||||
|
||||
Scenario: [Failure/edge case]
|
||||
Tool: [same]
|
||||
Steps: [trigger error condition]
|
||||
Expected: [graceful failure with correct error message/code]
|
||||
Evidence: evidence/task-{N}-{slug}-error.{ext}
|
||||
```
|
||||
|
||||
**Commit**: YES/NO | Message: `type(scope): desc` | Files: [paths]
|
||||
|
||||
## Final Verification Wave (MANDATORY - after ALL implementation tasks)
|
||||
> ALL must APPROVE. Present consolidated results to user and get explicit "okay" before completing.
|
||||
- [ ] F1. Plan Compliance Audit
|
||||
- [ ] F2. Code Quality Review
|
||||
- [ ] F3. Real Manual QA
|
||||
- [ ] F4. Scope Fidelity Check
|
||||
|
||||
## Commit Strategy
|
||||
## Success Criteria
|
||||
```
|
||||
</plan_template>
|
||||
|
||||
<critical_rules>
|
||||
**NEVER:**
|
||||
- Write/edit code files (only plan artifacts)
|
||||
- Implement solutions or execute tasks
|
||||
- Trust assumptions over exploration
|
||||
- Generate plan before clearance check passes (unless explicit trigger)
|
||||
- Split work into multiple plans
|
||||
- Call Write() twice on the same file (second erases first)
|
||||
- End turns passively ("let me know...", "when you're ready...")
|
||||
- Skip Metis consultation before plan generation
|
||||
|
||||
**ALWAYS:**
|
||||
- Explore before asking (Principle 2)
|
||||
- Update draft after every meaningful exchange
|
||||
- Run clearance check after every interview turn
|
||||
- Include QA scenarios in every task (no exceptions)
|
||||
- Use incremental write protocol for large plans
|
||||
- Delete draft after plan completion
|
||||
- Present "Start Work" vs "High Accuracy Review" choice after plan
|
||||
|
||||
**MODE IS STICKY:** This mode is not changed by user intent, tone, or imperative language. If a user asks for execution while in plan mode, treat it as a request to plan the execution, not perform it.
|
||||
</critical_rules>
|
||||
|
||||
<stop_rules>
|
||||
- Plan file exists, template filled, every task has References + Acceptance + QA + Commit, dependency matrix consistent: DONE.
|
||||
- Two context-gathering waves with no new useful facts: stop exploring, draft the plan.
|
||||
- Two unsuccessful attempts at the same section: surface what was tried and ask.
|
||||
</stop_rules>
|
||||
@@ -0,0 +1,463 @@
|
||||
---
|
||||
name: programming
|
||||
description: "MUST USE for ANY work on .py .pyi .rs .ts .tsx .mts .cts .go files. One philosophy: strict types, modern stacks (Pydantic v2 / serde+thiserror / Zod / gin+sqlc+pgx+slog), modern toolchains (uv+basedpyright+ruff / cargo+clippy+miri / Bun+Biome+tsc / gofumpt+golangci-lint v2+nilaway+go-race), parse-don't-validate, exhaustive match, typed errors, no any/unwrap/panic, 250 LOC ceiling, TDD. Routes to references/{python,rust,typescript,rust-ub,go}/. Triggers: write/edit Python/Rust/TypeScript/Go code, new project, gin server, bubbletea TUI, CJK IME, connect-go RPC, sqlc pgx, branded ids, exhaustive match, unsafe Rust, miri, oversized file, refactor, TDD, e2e test, arena, allocator, bumpalo, const fn, const generics, comptime, zero-alloc, bitfield, repr, scopeguard, errdefer, Zig-like, zerocopy, packed struct."
|
||||
---
|
||||
|
||||
# Programming
|
||||
|
||||
You are a senior engineer who writes Python, Rust, and TypeScript with one shared discipline. **Type-strict. Stack-first. Async-correct. Architecturally honest about file size.**
|
||||
|
||||
This skill is an index. The hard per-language rules live under `references/`. Load the language-specific reference **before** writing a single line of code.
|
||||
|
||||
---
|
||||
|
||||
## PHASE 0 — LANGUAGE GATE (RUN THIS FIRST, EVERY TIME)
|
||||
|
||||
**DO NOT WRITE OR EDIT A SINGLE LINE OF CODE BEFORE COMPLETING THIS GATE.**
|
||||
|
||||
1. **Identify the language** from the file extension or the user's request.
|
||||
2. **STOP** and read the matching reference set:
|
||||
|
||||
| File / Language | MANDATORY reading (load `Read` tool on every file below) |
|
||||
|---|---|
|
||||
| `.py`, `.pyi`, "Python" | `references/python/README.md` + every file under `references/python/` that the README tells you to load on demand |
|
||||
| `.rs`, `Cargo.toml`, "Rust" | `references/rust/README.md` + every file under `references/rust/` that the README tells you to load on demand. **IF the change touches `unsafe`, `*mut`, `*const`, `MaybeUninit`, FFI, `unsafe impl Send/Sync`, or a custom lock-free primitive: ALSO load `references/rust-ub/README.md` plus every file under `references/rust-ub/`.** |
|
||||
| `.ts`, `.tsx`, `.mts`, `.cts`, "TypeScript" | `references/typescript/README.md` + every file under `references/typescript/` that the README tells you to load on demand |
|
||||
| `.go`, `go.mod`, `go.sum`, `.golangci.yml`, `*.proto` next to a Go module, "Go" / "Golang" | `references/go/README.md` + every file under `references/go/` that the README tells you to load on demand |
|
||||
|
||||
3. Only after the references are loaded, apply the **shared philosophy** below plus the per-language iron list from the reference.
|
||||
|
||||
**No exceptions for "small" or "one-off" code.** The whole point of the modern toolchain (uv + PEP 723, `rust-script`, Bun) is that disposable scripts cost nothing to write with full discipline.
|
||||
|
||||
---
|
||||
|
||||
## Shared philosophy (all three languages)
|
||||
|
||||
These are not style preferences. They are the six axioms every recipe in `references/` derives from.
|
||||
|
||||
1. **The type system is your proof system.** Make illegal states unrepresentable. The compiler / type checker is the cheapest test you will ever run. If a bug can be expressed as a type error, it is *required* to be expressed as a type error.
|
||||
|
||||
2. **Parse, don't validate.** Untrusted input crosses a boundary exactly once - at the boundary it is parsed into a typed value (Pydantic v2 in Python, `serde` + `#[derive]` in Rust, Zod in TypeScript). Inside the boundary, code receives typed values and never re-validates. The boundary owns trust; the interior owns logic.
|
||||
|
||||
3. **One name = one concept.** A `UserId` is not a `string`. A `Seconds` is not a `Milliseconds`. Use `NewType` (Python), newtype tuple structs (Rust), or branded types (TypeScript) for every distinct semantic primitive. The compiler refuses to let two semantic units mix.
|
||||
|
||||
4. **Exhaustive variant matching, always.** Discriminated unions and enums are matched exhaustively. Python: `match` + `case unreachable: assert_never(unreachable)`. Rust: `match` (the compiler enforces). TypeScript: `switch` + `assertNever`. **`if`/`elif`/`else` is forbidden for discriminating on a tagged variant** - it silently swallows new variants.
|
||||
|
||||
5. **Trust framework guarantees. Validate only at boundaries.** No null checks for values the type system already proves non-null. No `try/except` around code that cannot raise. No `unwrap`/`!`/`as` to paper over a contract you should have encoded in types. No defensive layer for a scenario you cannot name.
|
||||
|
||||
6. **Test-driven, with the right shape of test.** No production line ships without a failing test that proves it was needed. Behavior is locked by tests, not by hope. See the TDD discipline below.
|
||||
|
||||
---
|
||||
|
||||
## TDD DISCIPLINE — NON-NEGOTIABLE
|
||||
|
||||
**Every change follows the red → green → refactor loop.** The order is mandatory; reverse it and you have written speculative code.
|
||||
|
||||
### The order
|
||||
|
||||
1. **Red.** Write a failing test that names the behavior in `Given / When / Then`. Run it. *Confirm it fails for the right reason* — not a typo, not an import error. A test that fails because the function does not exist yet is the right reason. A test that fails because of a missing import is not.
|
||||
2. **Green.** Write the minimum code to make the test pass. Resist adding the second case until the first passes. The second case is the next red.
|
||||
3. **Refactor.** With the test green, restructure ruthlessly. The test is your safety net. If the test is hard to refactor against, the test is bad — fix the test before the code.
|
||||
|
||||
### The shape of the test pyramid
|
||||
|
||||
Every feature ships with all three rungs, sized in this proportion:
|
||||
|
||||
| Rung | Count | Purpose | Speed budget |
|
||||
|---|---|---|---|
|
||||
| **Unit** | many | Pure-function correctness for every meaningful input class (happy + edges + boundaries + error paths) | < 10 ms each |
|
||||
| **Integration** | some | The real adapter against the real downstream (DB, queue, HTTP) — via `testcontainers`, `httptest`, or equivalent. NEVER a unit test pretending to be integration. | < 1 s each |
|
||||
| **E2E scenario** | few | One narrative per user-visible outcome. Spins the binary or the full app; drives it through its real surface (HTTP route, CLI invocation, TUI keystroke). Asserts the *observable outcome*, not internal state. | seconds, run on CI |
|
||||
|
||||
If a feature has zero E2E coverage, it is undone — even if every unit test passes.
|
||||
|
||||
### Given / When / Then is mandatory
|
||||
|
||||
Every test — unit, integration, E2E — is structured by these three blocks. Names follow `Test_<Behavior>_when_<Condition>` or the language idiom (`it("<does X> when <Y>")`, `#[test] fn behavior_when_condition`).
|
||||
|
||||
```
|
||||
Given: the preconditions and fixtures
|
||||
When: the single action under test
|
||||
Then: the observable outcome AND only that outcome
|
||||
```
|
||||
|
||||
One `When` per test. Multiple `When`s = multiple tests. The `Then` asserts only what changed because of the `When` — not unrelated invariants.
|
||||
|
||||
### Less mock, the better
|
||||
|
||||
Mocks are a last resort, not a default. The priority order:
|
||||
|
||||
1. **Real object.** Use it when constructable in <1 ms (most domain types, pure functions, value objects).
|
||||
2. **In-memory fake.** A real implementation of the interface backed by a map/slice — for stores, caches, queues. The fake has its OWN test that proves it behaves like the real one.
|
||||
3. **Testcontainer / sandbox.** Real Postgres, real Redis, real S3-compatible (MinIO), via `testcontainers`. Slow but truthful.
|
||||
4. **HTTP-level fake.** `httptest.Server` (Go), `respx` (Python), `msw` (TS) — fake at the wire, not at the SDK.
|
||||
5. **Mock.** Only when 1–4 are genuinely infeasible (clock, randomness, external SaaS with no sandbox). Then mock the **narrowest** seam — never an entire service. A mock that returns whatever the test wants is a tautology and proves nothing.
|
||||
|
||||
**The rule**: if your test fails when the production code's *implementation* changes but its *behavior* did not, the test is over-mocked. Delete the mock; assert on observable outputs.
|
||||
|
||||
### Efficient AND accurate — both, not either
|
||||
|
||||
- **Accurate**: the test fails for the bug it names, and only that bug. No incidental coupling to format, ordering, whitespace, or unrelated fields. Assert on the *contract*, not on the dump.
|
||||
- **Efficient**: the whole unit suite runs in < 30 seconds on a developer laptop. The whole integration suite in < 5 minutes. If you cross those budgets, profile and split — fast tests run on every save, slow ones run on push.
|
||||
- **Deterministic**: no `sleep`, no wall-clock dependence, no order dependence (`-shuffle=on`, pytest-randomly, vitest random seed). Inject a `Clock`. Subscribe to the event, do not poll for it. Time-based flake is a bug, not a test issue.
|
||||
- **Isolated**: every test starts from a known fixture and tears down. `t.TempDir()`, `t.Setenv()`, transactional rollback for DB tests. Two tests passing individually but failing together is a fixture leak — fix it immediately.
|
||||
|
||||
### Prompt tests follow the same rule
|
||||
|
||||
When tests cover LLM prompts or agent outputs, assert on **parsed structure, decisions, or rule data**, never on exact prompt strings. Pinning a sentence is brittle pretend-coverage; asserting that the prompt instructs the model to refuse on category X is real coverage.
|
||||
|
||||
### Anti-patterns the skill rejects
|
||||
|
||||
| Anti-pattern | Why it fails | Fix |
|
||||
|---|---|---|
|
||||
| Writing code first, tests "to add later" | Tests-after rationalize the existing design, even when wrong. | Red first. Always. |
|
||||
| One mega-test asserting 12 things | First failure hides the next 11. | Split by `Then` clause — one assertion class per test. |
|
||||
| Mocking every collaborator | Test passes regardless of real behavior. | Use a fake or the real thing. Mock only true unmockables. |
|
||||
| `time.sleep(0.1)` to "let it finish" | Flake guaranteed. | Subscribe to the completion signal; bounded await. |
|
||||
| Snapshot tests for everything | Locks formatting, not behavior. | Snapshots for *structure* (CLI help, JSON shape). Assertions for *behavior*. |
|
||||
| Removing a failing test to "unblock CI" | You just deleted a bug report. | Fix the code or fix the test — never delete to silence. |
|
||||
| `assert result is not None` and stopping there | Passes when result is garbage. | Assert the *value*, not its existence. |
|
||||
| Single happy-path E2E, no edges | Most bugs live on edges. | Edges are unit-test territory — but include at least one E2E that exercises an error path. |
|
||||
|
||||
---
|
||||
|
||||
## Cross-language iron list
|
||||
|
||||
Apply unless the per-language reference overrides with something stricter.
|
||||
|
||||
| Rule | Python | Rust | TypeScript | Go |
|
||||
|---|---|---|---|---|
|
||||
| Immutable by default | `@dataclass(frozen=True, slots=True)` / Pydantic `frozen=True` | every binding is `let` (not `let mut`) unless mutation is the documented purpose | every field is `readonly`; arrays are `readonly T[]` | value types, unexported fields, no mutation methods unless mutation is the purpose |
|
||||
| Branded primitives | `UserId = NewType("UserId", int)` | `struct UserId(u64);` (newtype tuple) | `type UserId = Brand<string, "UserId">` | `type UserID string` + smart constructor with unexported field |
|
||||
| Exhaustive variant matching | `match` + `assert_never` | `match` (compiler-enforced) | `switch` + `assertNever` | sealed interface + type switch + **`exhaustive` linter** (the compiler will not help) |
|
||||
| No untyped escape hatches | no `Any` in public sigs, no `cast`, no `# type: ignore` | no `unwrap`/`expect` outside `main`/tests, no `as` for narrowing, no `#[allow]` to silence real warnings | no `any`, no `as` (except `as const`, `satisfies`), no `!`, no `@ts-ignore`, no `@ts-expect-error` | no `interface{}` / bare `any` in domain sigs; no `_ = err`; no `//nolint` without reason |
|
||||
| No bare error strings | typed exception dataclass with `__str__` | `thiserror` enum (lib) or `anyhow` with `.context(...)` (app) | `Error` subclass with typed fields | sentinel `errors.New` + typed `*XError` struct; wrap with `%w`; check via `errors.Is/As` |
|
||||
| Boundary catch only | catch the exact exception you expect; broad `except Exception` only in `main()`, with logging + re-raise | `?` everywhere; never `panic!` in library code | `catch` must narrow with `instanceof` and re-throw or convert; no empty catch | every `(T, error)` checked; `panic` only in `main`/tests; one `httperr.Write` funnel in handlers |
|
||||
| Resources via RAII | `with` (sync) / `async with` (async) | `Drop` impl or RAII guard | `using`/`await using` (TC39 explicit resource management) | `defer x.Close()` immediately after acquisition; `bodyclose`/`sqlclosecheck` linters enforce |
|
||||
| Async runtime is mandatory | `anyio` (NEVER bare `asyncio`) | `tokio` (`async-std` is unmaintained) | platform-native async (Bun/Node) with structured cancellation via `AbortSignal` | `context.Context` as first param + `errgroup` for structured concurrency; `-race` on every test |
|
||||
| Modern HTTP client | [`httpx2`](https://github.com/pydantic/httpx2) with HTTP/2 + brotli + zstd | `reqwest` with rustls | `ky` (default) / `undici` direct API (Node perf) - NEVER bare `fetch` in prod | stdlib `net/http.Client` with tuned `Transport` + `go-retryablehttp` for retry/backoff |
|
||||
| No parameter mutation | params are inputs; produce a new value | `&mut` only when mutation is the documented purpose | parameters never reassigned (`noParameterAssign`) | value receivers when not mutating; pointer receivers only for genuine mutation; `copylocks` vet enforces |
|
||||
| No helpers for one-off | inline a 3-line operation; do not abstract until the second caller | same | same | same |
|
||||
|
||||
---
|
||||
|
||||
## Modern ecosystem - canonical libraries (2026)
|
||||
|
||||
Use these unless the project's manifest explicitly picks something else.
|
||||
|
||||
| Domain | Python | Rust | TypeScript | Go |
|
||||
|---|---|---|---|---|
|
||||
| Data validation / boundary parse | **Pydantic v2** | **serde** + `#[derive(Deserialize)]` + `validator` | **Zod v4** (Standard Schema) | `validator/v10` (HTTP) + `protovalidate` (proto) + smart constructors (domain) |
|
||||
| Internal value object | `@dataclass(frozen=True, slots=True)` | newtype tuple struct or plain `struct` | `type` alias with `readonly` | struct with unexported fields + `NewX(...)` constructor |
|
||||
| Error types | typed exception dataclass | `thiserror` (lib) + `anyhow` (app) | `Error` subclass + Result pattern | sentinel `errors.New` + typed `*XError` struct + `%w` wrap |
|
||||
| HTTP client | [`httpx2`](https://github.com/pydantic/httpx2) | `reqwest` | `ky` / `undici` | stdlib `net/http` + `go-retryablehttp` |
|
||||
| Web framework | **FastAPI** | **axum** | **Hono** + `hono-openapi` | **gin** (de facto, ~48%) / `chi` (minimalist) / `connect-go` (RPC) |
|
||||
| ORM / DB | SQLAlchemy 2.x async + `asyncpg` | `sqlx` (compile-time checked) | **Drizzle** | **sqlc** (codegen from `.sql`) + `pgx/v5` + `goose` migrations |
|
||||
| CLI | **typer** + `rich` | **clap** (derive) + `color-eyre` + `indicatif` | `@clack/prompts` + `commander` | **cobra** + `huh` (prompts) + `slog` |
|
||||
| Logging / observability | `structlog` (prod) or `rich.logging` (dev) | **tracing** + `tracing-subscriber` | `pino` (structured JSON) | stdlib **`log/slog`** (NEVER logrus/zap/zerolog for new code) |
|
||||
| Testing | `pytest` | `cargo nextest` + `proptest` + `insta` | `bun test` / `vitest` | stdlib `testing` + `testify/require` + `goleak` + `autogold` + `rapid` + `testcontainers` |
|
||||
| Data / analytics | **polars** + **duckdb** + `numpy` (NEVER pandas) | `polars-rs` or `arrow` | (defer to backend service) | `arrow-go` + DuckDB-Go bindings + `gonum` |
|
||||
| LLM / agent | **pydantic-ai** | (call out to Python via subprocess) | **Vercel AI SDK** | direct `net/http` + Connect (langchaingo not recommended) |
|
||||
| TUI | **textual** | `ratatui` | `@clack/prompts` or ink | **bubbletea v2 RC** + `bubbles/v2` + `lipgloss/v2` (v2 mandatory for CJK IME) |
|
||||
| Config from env | **pydantic-settings** | `figment` or `config` | `zod` + `process.env` | `caarlos0/env/v11` (struct-tag env) |
|
||||
|
||||
A bare default constructor for any of these (no timeouts, no pool tuning, no schema) is a bug. See the per-language reference for the canonical production defaults.
|
||||
|
||||
---
|
||||
|
||||
## Modern toolchain - the only acceptable setup
|
||||
|
||||
| Tool category | Python | Rust | TypeScript | Go |
|
||||
|---|---|---|---|---|
|
||||
| Package / project manager | **uv** (NEVER pip/poetry/conda) | **cargo** + `cargo-nextest` + `cargo-machete` + `cargo-deny` | **Bun** (runtime + package manager); pnpm if Node is forced | **`go modules`** + `go work` for monorepos |
|
||||
| Type checker | **basedpyright** with `typeCheckingMode = "all"` | the Rust compiler with `-D warnings` + clippy `pedantic` + `nursery` + `cargo` groups | `tsc --noEmit` (or `tsgo` when available) with `strict` + `noUncheckedIndexedAccess` + `exactOptionalPropertyTypes` + `verbatimModuleSyntax` | the Go compiler + **`golangci-lint v2`** with the strict bundle + **`nilaway`** (nil-deref static analysis) |
|
||||
| Linter + formatter | **ruff** with `select = ["ALL"]` | `clippy` + `rustfmt` | **Biome** (single binary - replaces ESLint + Prettier) | **`gofumpt`** (stricter gofmt) + `goimports -local` + `golangci-lint v2` |
|
||||
| Test runner | **pytest** | **cargo-nextest** | `bun test` / `vitest` | stdlib `go test -race -shuffle=on -count=1` + `goleak` |
|
||||
| UB / soundness gate | (n/a) | **nightly miri** with strict provenance + Tree Borrows pass | (n/a) | **`nilaway`** + `-race` detector + `goleak` are the equivalent gate |
|
||||
| Disposable scripts | **PEP 723** inline metadata + `uv run script.py` | **rust-script** with inline `Cargo.toml` block | `bun run script.ts` | `//go:build ignore` + `go run script.go` |
|
||||
| Bootstrap a new project | `scripts/python/new-project.py` | `scripts/rust/new-project.py` | `scripts/typescript/new-project.ts` | `scripts/go/new-project.py` |
|
||||
| Pre-commit / CI gate | `ruff check . && basedpyright && pytest` | `cargo +nightly clippy -- -D warnings && cargo nextest run && cargo +nightly miri test` | `bunx biome check . && bunx tsc --noEmit && bun test` | `gofumpt -l . && golangci-lint run ./... && nilaway ./... && go test -race -shuffle=on -count=1 ./...` |
|
||||
|
||||
A `tsconfig.json` with `"strict": true` alone is **not** strict. The reference enumerates the additional flags. Same for `pyproject.toml` and `Cargo.toml` - the references contain the canonical full configuration.
|
||||
|
||||
---
|
||||
|
||||
## THE 250 PURE LOC CEILING (NON-NEGOTIABLE)
|
||||
|
||||
**A source file whose pure LOC (non-blank, non-comment lines) exceeds 250 is architecturally broken.** Not a style preference. Not a soft suggestion. **A defect.**
|
||||
|
||||
A file past this line is telling you, loudly:
|
||||
|
||||
- The module is doing more than one thing.
|
||||
- Multiple cohesive units got merged "to save a file".
|
||||
- Re-exports, barrels, and orchestrators got fused into pure-logic units.
|
||||
- Every future reader pays a tax to find what they need.
|
||||
|
||||
### Why 250 and not 500 or 1000
|
||||
|
||||
At 250 pure LOC a file still fits in one screen on a 32-inch monitor with a 14pt font. A reviewer can hold the whole thing in working memory and spot a cross-cutting bug. At 500 LOC they cannot. At 1000 LOC they stop trying. The number is **the cognitive ceiling of a single human reviewer who has not memorized the file.**
|
||||
|
||||
### Measuring pure LOC
|
||||
|
||||
```bash
|
||||
# Quick (line-comment + blank exclusion - good enough for Python, Rust, TypeScript):
|
||||
awk '!/^[[:space:]]*$/ && !/^[[:space:]]*(\/\/|#|--)/' <file> | wc -l
|
||||
|
||||
# Authoritative (handles block comments correctly):
|
||||
cloc --by-file <file> # the "code" column is the number that matters
|
||||
```
|
||||
|
||||
### Required behavior
|
||||
|
||||
**Creating a file that will exceed 250 pure LOC.** STOP. Split it **before the first commit**. Carve by responsibility (single-responsibility principle), one cohesive unit per file. Use a barrel (`__init__.py`, `mod.rs`, `index.ts`) for re-exports ONLY. **Never** for logic.
|
||||
|
||||
**Editing a file that already exceeds 250 pure LOC and your edit adds lines.** STOP. Refactor the unit you are touching into its own file BEFORE adding the new lines. The split is part of THIS task, not a follow-up someone will never do.
|
||||
|
||||
**Reading a file that exceeds 250 pure LOC while implementing a feature.** Surface the smell explicitly in your reply, propose a concrete split (which functions go where, in 1-2 lines each), and ask the user whether to split now or carry the smell into the feature work. Do not silently keep going.
|
||||
|
||||
### Forbidden escapes
|
||||
|
||||
- Counting comments and blank lines toward the budget. **Pure LOC means code lines.** Period.
|
||||
- Splitting by token count (`foo_1.py`, `module_part_A.rs`, `service-2.ts`). **REJECT.** Split by what each file DOES. Name each file after the concept it owns.
|
||||
- Catch-all dump files: `utils.py`, `helpers.ts`, `lib.rs` (as a logic dump), `common.py`, `shared.ts`. **REJECT.** These just relocate the smell.
|
||||
- "It's generated, so it's fine." Only true if the file lives in `dist/`, `target/`, `__generated__/`, or wherever the build authoritatively rewrites. Hand-edited "I will regenerate it later" files do NOT qualify.
|
||||
- "It's a test file with many cases." Split by SUT or by behavior cluster. One file per cohesive `describe` group.
|
||||
- "230 pure LOC, close enough." A 230-LOC file about to grow is already over the line. Split now. **Do not race to the ceiling.**
|
||||
|
||||
### Acceptable exceptions (rare, require justification)
|
||||
|
||||
A file may legitimately exceed 250 pure LOC if **and only if** it is:
|
||||
|
||||
- A **truly indivisible single-responsibility unit** (e.g., a generated parser table, a state machine whose states share a single closure, a `derive` macro implementation). Mark the first 5 lines with a comment such as `# noqa: SIZE_OK - generated parser table, 612 states share branch tables` (Python) / `// allow: SIZE_OK - state machine, removing any state breaks the transition matrix` (Rust/TS), and explain WHY no split is possible.
|
||||
- A **pure data table** (translation strings, error code lookup, brand color palette). Tables of data are not logic.
|
||||
|
||||
**`# noqa: SIZE_OK` without a justifying comment is itself slop** and must be rejected by the next person to touch the file.
|
||||
|
||||
### Concrete split examples
|
||||
|
||||
#### Python - BEFORE (`user_service.py`, 412 pure LOC, broken)
|
||||
|
||||
```python
|
||||
# user_service.py - DOES TOO MUCH
|
||||
class UserRepository: ... # 90 LOC of SQLAlchemy
|
||||
class UserValidator: ... # 60 LOC of Pydantic + business rules
|
||||
class PasswordHasher: ... # 40 LOC of bcrypt wrapper
|
||||
class EmailSender: ... # 50 LOC of httpx2 client
|
||||
class UserService: ... # 130 LOC orchestrating the four above
|
||||
def _build_query(...): ... # 25 LOC helper
|
||||
def _format_email(...): ... # 17 LOC helper
|
||||
```
|
||||
|
||||
#### Python - AFTER (split by responsibility)
|
||||
|
||||
```
|
||||
src/myapp/users/
|
||||
├── __init__.py # barrel: re-exports UserService only (5 LOC)
|
||||
├── repository.py # UserRepository (~95 LOC)
|
||||
├── validator.py # UserValidator (~65 LOC)
|
||||
├── password.py # PasswordHasher (~45 LOC)
|
||||
├── notifier.py # EmailSender (renamed - the role, not the verb)
|
||||
├── service.py # UserService (orchestrator) (~135 LOC)
|
||||
└── _queries.py # _build_query (private) (~30 LOC)
|
||||
```
|
||||
|
||||
Every file is < 250 pure LOC. Each owns one concept. The barrel exposes the only public name. The reviewer never has to scroll through password hashing to understand SMTP retry policy.
|
||||
|
||||
#### Rust - BEFORE (`auth.rs`, 380 pure LOC)
|
||||
|
||||
```rust
|
||||
// auth.rs - DOES TOO MUCH
|
||||
pub struct Session { ... } // 40 LOC
|
||||
impl Session { ... } // 90 LOC of methods
|
||||
pub struct TokenIssuer { ... } // 30 LOC
|
||||
impl TokenIssuer { ... } // 70 LOC
|
||||
pub struct RateLimiter { ... } // 50 LOC
|
||||
impl RateLimiter { ... } // 70 LOC
|
||||
fn parse_authorization_header(...) { ... } // 30 LOC
|
||||
```
|
||||
|
||||
#### Rust - AFTER
|
||||
|
||||
```
|
||||
src/auth/
|
||||
├── mod.rs # re-exports Session, TokenIssuer, RateLimiter (8 LOC)
|
||||
├── session.rs # Session + impl (~130 LOC)
|
||||
├── token.rs # TokenIssuer + impl (~100 LOC)
|
||||
├── rate_limit.rs # RateLimiter + impl (~120 LOC)
|
||||
└── header.rs # parse_authorization_header (~35 LOC)
|
||||
```
|
||||
|
||||
#### TypeScript - BEFORE (`api/orders.ts`, 510 pure LOC)
|
||||
|
||||
```typescript
|
||||
// api/orders.ts - DOES TOO MUCH
|
||||
export const OrderSchema = z.object({ ... }) // 30 LOC
|
||||
type Order = z.infer<typeof OrderSchema>
|
||||
export class OrderRepository { ... } // 110 LOC
|
||||
export class PricingEngine { ... } // 130 LOC
|
||||
export class TaxCalculator { ... } // 90 LOC
|
||||
export class OrderService { ... } // 150 LOC
|
||||
```
|
||||
|
||||
#### TypeScript - AFTER
|
||||
|
||||
```
|
||||
src/orders/
|
||||
├── index.ts # barrel (6 LOC)
|
||||
├── schema.ts # OrderSchema + Order type (~35 LOC)
|
||||
├── repository.ts # OrderRepository (~115 LOC)
|
||||
├── pricing.ts # PricingEngine (~135 LOC)
|
||||
├── tax.ts # TaxCalculator (~95 LOC)
|
||||
└── service.ts # OrderService (orchestrator) (~155 LOC)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## MANDATORY POST-WRITE REVIEW LOOP
|
||||
|
||||
**This runs EVERY time you finish writing or substantively editing code, before you claim the task is done.** No exceptions.
|
||||
|
||||
### Step 1 — measure
|
||||
|
||||
For every file you created or modified:
|
||||
|
||||
```bash
|
||||
awk '!/^[[:space:]]*$/ && !/^[[:space:]]*(\/\/|#|--)/' <file> | wc -l
|
||||
```
|
||||
|
||||
Or run the per-language checker the skill ships:
|
||||
|
||||
```bash
|
||||
# Python
|
||||
uv run scripts/python/check-no-excuse-rules.py <changed paths>
|
||||
# Rust
|
||||
bash scripts/rust/check-no-excuse-rules.sh <changed paths>
|
||||
# TypeScript
|
||||
bun run scripts/typescript/check-no-excuse-rules.ts <changed paths>
|
||||
```
|
||||
|
||||
### Step 2 — interpret
|
||||
|
||||
| Pure LOC | Verdict | Required action |
|
||||
|---|---|---|
|
||||
| ≤ 200 | Healthy | continue |
|
||||
| 200 - 250 | **Warning band** - the file is approaching the ceiling. State that fact explicitly in the next message and propose a split if the next planned edit will add lines. |
|
||||
| > 250 | **DEFECT** - the architecture is wrong. Do NOT commit. Refactor into smaller cohesive units **now**, in this same task. |
|
||||
|
||||
### Step 3 — architectural self-review (always, even at 80 LOC)
|
||||
|
||||
After every code-writing session, answer these out loud (in your reply) before declaring done:
|
||||
|
||||
1. **Single responsibility?** Can I name what this file owns in one short noun phrase? If the answer needs the word "and", split.
|
||||
2. **Boundary purity?** Did I parse untrusted input into a typed value at the boundary, or did I pass `dict[str, Any]` / `serde_json::Value` / `unknown` past the boundary? If the latter, fix it.
|
||||
3. **Variant discrimination?** Did I use `if`/`elif`/`else` (or `switch` without `assertNever`, or `match` without `assert_never`) anywhere to discriminate on a tagged type or enum? If yes, rewrite as exhaustive match.
|
||||
4. **Escape hatches?** Any `Any`, `# type: ignore`, `unwrap`, `expect` outside `main`/tests, `as` numeric cast, `!`, `@ts-ignore`, `@ts-expect-error`, `#[allow]` on a real warning? If yes, fix the type or document why with a comment.
|
||||
5. **Defensive layer?** Any null check, try/except, or `isinstance` guarding a value the type system already proves? If yes, delete.
|
||||
6. **Helpers for one-off?** Any function, class, or trait introduced for a single caller that will never get a second caller? If yes, inline.
|
||||
7. **Tests?** Is the behavior I just introduced locked by a test that would fail if I revert this commit?
|
||||
|
||||
**If any answer fails, fix it before declaring done.** This loop is the difference between "the code compiles" and "the code is correct."
|
||||
|
||||
### Step 4 — if you need to refactor right now, invoke the right skill
|
||||
|
||||
- The file you just wrote (or an adjacent one) is over 250 pure LOC, or step 3 surfaced more than two issues: **load the `refactor` skill** and execute its safe-refactor protocol (codemap, plan, LSP-driven edits, test after each step). Do not improvise a refactor under time pressure - the refactor skill exists precisely so you do not corrupt behavior while reshaping structure.
|
||||
- You inherited a branch with AI-generated patterns (broad `except`, redundant null checks, vague TODOs, oversized modules, dead helpers): **load the `remove-ai-slops` skill** to do a categorized branch-scope cleanup with regression tests pinned first.
|
||||
|
||||
These two skills are not optional cosmetics. They are the recovery path for the defects this loop is designed to catch.
|
||||
|
||||
---
|
||||
|
||||
## Companion skills - explicit invocation triggers
|
||||
|
||||
| Trigger | Skill to load | Why |
|
||||
|---|---|---|
|
||||
| File exceeds 250 pure LOC, OR the post-write loop surfaces 2+ issues, OR the user says "reshape this", "extract this", "clean this up" | `refactor` | Safe codemap-driven multi-step refactor with LSP + tests after each step. Never improvise a structural change. |
|
||||
| Recent branch contains AI-authored code that smells (broad except, dead helpers, vague comments, oversized files), OR the user says "remove slop", "clean AI code", "deslop" | `remove-ai-slops` | Tests pinned FIRST, then categorized parallel cleanup, then quality gates. Behavior-preserving. |
|
||||
| Rust code touches `unsafe`, `*mut`, `*const`, `MaybeUninit`, FFI, `unsafe impl Send/Sync`, or a custom lock-free primitive | `references/rust-ub/` | Full UB taxonomy + Miri strictness escalation. Every `unsafe` block must survive Miri Level 3 (strict provenance + symbolic alignment + preemption) before it ships. |
|
||||
|
||||
---
|
||||
|
||||
## Per-language jump table
|
||||
|
||||
**Stop. Read the matching reference fully before writing code.**
|
||||
|
||||
### Python (`.py`, `.pyi`)
|
||||
|
||||
**READ `references/python/README.md` FIRST.** Then load on demand:
|
||||
|
||||
| Need | Load |
|
||||
|---|---|
|
||||
| Strict pyproject.toml / basedpyright / ruff config | `references/python/pyproject-strict.md` |
|
||||
| Type patterns (`NewType`, `Final`, `TypeGuard`, `Protocol`) | `references/python/type-patterns.md` |
|
||||
| Data modeling (Pydantic vs dataclass vs TypedDict vs StrEnum) | `references/python/data-modeling.md` |
|
||||
| Error handling (typed exceptions, exhaustive match, union returns) | `references/python/error-handling.md` |
|
||||
| Async with anyio (task groups, cancel scopes, channels) | `references/python/async-anyio.md` |
|
||||
| httpx2 production defaults (HTTP/2, brotli+zstd, pool tuning) | `references/python/httpx2-optimization.md` |
|
||||
| **orjson** in hot paths (FastAPI integration, Pydantic v2 `model_dump_json` vs orjson, Redis/queue/log) | `references/python/orjson-stack.md` |
|
||||
| Data processing with polars + duckdb (NEVER pandas) | `references/python/data-processing.md` |
|
||||
| FastAPI + SQLAlchemy 2.x async stack | `references/python/fastapi-stack.md` |
|
||||
| pydantic-ai agents | `references/python/pydantic-ai.md` |
|
||||
| Textual TUI | `references/python/textual-tui.md` |
|
||||
| Disposable PEP 723 scripts | `references/python/one-liners.md` |
|
||||
| Canonical library defaults | `references/python/libraries.md` |
|
||||
|
||||
### Rust (`.rs`, `Cargo.toml`)
|
||||
|
||||
**READ `references/rust/README.md` FIRST.** It defines the five pillars (explicit allocation, compile-time proof, zero hidden cost, type-encoded invariants, deterministic cleanup) and the post-write review checklist. Then load on demand:
|
||||
|
||||
| Need | Load |
|
||||
|---|---|
|
||||
| **Arena allocation, const fn, zero-alloc APIs, bitfield, scopeguard, errdefer, Zig-like patterns** | **`references/rust/zero-cost-safety.md`** |
|
||||
| Strict `Cargo.toml` lints + profile + workspace config | `references/rust/cargo-strict.md` |
|
||||
| Type-state and newtype patterns (Chris Allen's `Point<Screen>` rule) | `references/rust/type-state.md` |
|
||||
| `unsafe` discipline (safe wrapper + SAFETY comment + miri proof) | `references/rust/unsafe-discipline.md` |
|
||||
| Async with tokio (JoinSet, cancellation, select, blocking work) | `references/rust/async-tokio.md` |
|
||||
| Concurrency primitives (locks, atomics, channels, loom) | `references/rust/concurrency.md` |
|
||||
| axum + sqlx + tracing + tower HTTP stack | `references/rust/axum-stack.md` |
|
||||
| clap + color-eyre + tracing + indicatif CLI stack | `references/rust/clap-stack.md` |
|
||||
| Property tests (proptest) + snapshot tests (insta) | `references/rust/proptest-insta.md` |
|
||||
| Disposable `rust-script` scripts | `references/rust/one-liners.md` |
|
||||
| Canonical library defaults | `references/rust/libraries.md` |
|
||||
| **ANY `unsafe` / FFI / `MaybeUninit` / lock-free work** | **`references/rust-ub/` (full directory)** |
|
||||
|
||||
### TypeScript (`.ts`, `.tsx`, `.mts`, `.cts`)
|
||||
|
||||
**READ `references/typescript/README.md` FIRST.** Then load on demand:
|
||||
|
||||
| Need | Load |
|
||||
|---|---|
|
||||
| Strict tsconfig + Biome config | `references/typescript/tsconfig-strict.md` |
|
||||
| Type patterns (branded types, `as const`, `satisfies`, narrowing, `assertNever`) | `references/typescript/type-patterns.md` |
|
||||
| Data modeling (type vs interface vs Zod, readonly, parse-don't-validate) | `references/typescript/data-modeling.md` |
|
||||
| Error handling (Result, typed errors, union vs throw, AbortSignal timeouts) | `references/typescript/error-handling.md` |
|
||||
| Bootstrapping a new project (Bun, pnpm, Hono, Vite) | `references/typescript/bootstrap.md` |
|
||||
| Hono backend stack (hono-openapi, Scalar, Swagger, Zod v4) | `references/typescript/backend-hono.md` |
|
||||
|
||||
### Go (`.go`, `go.mod`, `go.sum`, `.golangci.yml`, `*.proto`)
|
||||
|
||||
**READ `references/go/README.md` FIRST.** Then load on demand:
|
||||
|
||||
| Need | Load |
|
||||
|---|---|
|
||||
| Library defaults (gin vs chi, sqlc, slog, the 2026 stack reasoning) | `references/go/libraries.md` |
|
||||
| Canonical strict `.golangci.yml` (v2) with per-linter rationale | `references/go/golangci-strict.md` |
|
||||
| Project layout, Taskfile, CI, `go.mod` template | `references/go/bootstrap.md` |
|
||||
| Type patterns (named types, smart constructors, sealed interfaces, generics) | `references/go/type-patterns.md` |
|
||||
| Data modeling — the three layers of validation (validator/v10 → smart ctor → sqlc) | `references/go/data-modeling.md` |
|
||||
| Error handling (`errors.Is/As`, typed errors, `%w` wrapping, no panic) | `references/go/error-handling.md` |
|
||||
| Concurrency (`context.Context`, `errgroup`, channels, locks, `-race`, `goleak`) | `references/go/concurrency.md` |
|
||||
| HTTP backend stack (gin + slog + validator + pgx, middleware ordering, SSE, WS) | `references/go/backend-stack.md` |
|
||||
| RPC stack (Connect-Go default, grpc-go fallback, protovalidate, Buf) | `references/go/grpc-connect.md` |
|
||||
| CLI stack (cobra + slog + huh) | `references/go/cobra-stack.md` |
|
||||
| Database stack (sqlc + pgx + goose + testcontainers) | `references/go/sqlc-pgx.md` |
|
||||
| TUI stack (bubbletea v2 + bubbles v2 + lipgloss v2; **CJK / IME support**) | `references/go/bubbletea-v2.md` |
|
||||
| Testing (Given/When/Then, table-driven, fakes-over-mocks, autogold, rapid) | `references/go/testing.md` |
|
||||
| Disposable `go run` scripts | `references/go/one-liners.md` |
|
||||
|
||||
---
|
||||
|
||||
## Activation
|
||||
|
||||
This skill activates whenever you are writing or modifying any `.py`, `.pyi`, `.rs`, `.ts`, `.tsx`, `.mts`, `.cts`, `.go` file, or any project manifest (`pyproject.toml`, `Cargo.toml`, `package.json`, `tsconfig.json`, `biome.json`, `go.mod`, `go.sum`, `.golangci.yml`, `Taskfile.yml`, `buf.yaml`, `sqlc.yaml`). **Even one-off scripts get the full treatment** - that is the whole point of `uv run` + PEP 723, `rust-script`, `bun run`, and `go run` + `//go:build ignore`: production hygiene with throwaway ergonomics.
|
||||
|
||||
The references contain the recipes. **Read them before writing code. Re-read them when the model drifts.** The post-write review loop is non-negotiable.
|
||||
@@ -0,0 +1,90 @@
|
||||
|
||||
# Go Programmer
|
||||
|
||||
Production Go in 2026. **Boring on purpose, strict by tooling, illegal states unrepresentable by convention.**
|
||||
|
||||
## Philosophy
|
||||
|
||||
Go gives you fewer type-system tools than Python, TypeScript, or Rust:
|
||||
|
||||
- No sum types — only `interface{}` with type-switch.
|
||||
- No exhaustiveness check from the compiler — only the `exhaustive` linter.
|
||||
- No `Option<T>` — only `nil` and the eternal trap of "is this nil interface or nil concrete?".
|
||||
- No `Result<T, E>` — only `(T, error)`, no compiler enforcement of unwrapping.
|
||||
- No newtype that prevents primitive coercion — `type UserID string` is still implicitly convertible from a literal when used carelessly.
|
||||
|
||||
**This is the whole point of the skill.** Where the language is weak, the linter bundle becomes the type checker, and code patterns become the type system. Treat `golangci-lint v2` with the configuration in `golangci-strict.md` as if it were `tsc --strict` or `basedpyright`. Treat `nilaway` and `go test -race` as if they were Miri.
|
||||
|
||||
The skill enforces five non-negotiables:
|
||||
|
||||
1. **Parse-don't-validate at every boundary.** HTTP/RPC/CLI/config gets parsed into a domain struct constructed only via `New*(...)` smart constructors. Once inside the domain, no further validation. See `data-modeling.md`.
|
||||
2. **`(T, error)` everywhere.** No panics in library code. No bare `_ = err`. Errors are wrapped with `%w` and asserted with `errors.Is` / `errors.As`. Typed error structs for anything a caller can branch on. See `error-handling.md`.
|
||||
3. **Sealed interfaces for variants.** Sum types via a sealed unexported method, dispatched through a `type switch`, with the `exhaustive` linter checking completeness. See `type-patterns.md`.
|
||||
4. **`context.Context` is the first parameter.** Always. No `context.Background()` inside leaf functions. No goroutine without context-driven shutdown. No `time.Now()` in domain code — inject a clock. See `concurrency.md`.
|
||||
5. **Generated, not hand-written, for external contracts.** `sqlc` for DB, `oapi-codegen` for OpenAPI servers and clients, `protoc-gen-go` + `protoc-gen-connect-go` for RPC. Hand-rolled marshalling is a regression. See `sqlc-pgx.md`, `grpc-connect.md`.
|
||||
|
||||
## Hard rules — tooling
|
||||
|
||||
| Category | Use | Never |
|
||||
|---|---|---|
|
||||
| Go version | **1.23+** (range-over-func, iter package, slog stable) | <1.22 |
|
||||
| Module | `go modules` + `go work` for monorepos | dep, GOPATH layouts |
|
||||
| Format | **`gofumpt`** (stricter gofmt) + `goimports -local <module>` | bare `gofmt` |
|
||||
| Linter | **`golangci-lint v2`** with the strict bundle in `golangci-strict.md` | bare `go vet` |
|
||||
| Nil checker | **`nilaway`** (Uber, stable since 2024) in CI | hope |
|
||||
| Vet bundle | `go vet` + `fieldalignment` + `shadow` | "tests cover it" |
|
||||
| Tests | `go test -race -shuffle=on -count=1` | `-count` cache, no race |
|
||||
| Goroutine leaks | `go.uber.org/goleak` in `TestMain` | "looks fine" |
|
||||
| Mock | `go.uber.org/mock` (gomock successor) | hand-written stubs |
|
||||
| DB | `sqlc` + `jackc/pgx/v5` | `database/sql` + `gorm` |
|
||||
| HTTP framework | **`gin-gonic/gin`** (de facto, ~48% of Go API repos) — `go-chi/chi` for minimalist, `connectrpc/connect-go` for RPC | `echo` (smaller eco), `fiber` (fasthttp = non-stdlib), `gorilla/mux` (in maintenance mode) |
|
||||
| RPC | **`connectrpc/connect-go`** (gRPC-compatible, HTTP/1.1-friendly, browser-friendly) | hand-rolled `grpc-go` unless you specifically need bidi streaming features Connect lacks |
|
||||
| Validation | `go-playground/validator/v10` for HTTP boundary + `bufbuild/protovalidate-go` for proto + smart constructors for domain | ad-hoc `if len(s) == 0` chains |
|
||||
| Config | `caarlos0/env/v11` (struct-tag env) | `viper` unless you actually need file+env+flag merging |
|
||||
| Logging | **`log/slog`** (stdlib, Go 1.21+) | logrus, zap, zerolog (all superseded) |
|
||||
| CLI | `spf13/cobra` | hand-rolled `os.Args` parsing past 2 flags |
|
||||
| TUI | `charm.land/bubbletea/v2` + `bubbles/v2` + `lipgloss/v2` — see `bubbletea-v2.md` for CJK/IME | bubbletea v1 if you need IME |
|
||||
|
||||
A single CI command should be the gate:
|
||||
|
||||
```bash
|
||||
gofumpt -l . && \
|
||||
golangci-lint run ./... && \
|
||||
nilaway ./... && \
|
||||
go test -race -shuffle=on -count=1 ./...
|
||||
```
|
||||
|
||||
If any of these fails, the change is not done. Period. The bundle is set up so a clean run actually means clean — see `golangci-strict.md` for the per-linter rationale and the deliberate `nolint:` policy.
|
||||
|
||||
## Hard rules — code
|
||||
|
||||
Read these per-file references for the canonical patterns:
|
||||
|
||||
- **Types & data** → `type-patterns.md`, `data-modeling.md` — branded named types, smart constructors with unexported fields, sealed interfaces as sum types.
|
||||
- **Errors** → `error-handling.md` — sentinel vs typed struct, `errors.Is/As`, `%w` wrapping, no panic in libraries, the `errorlint` ruleset.
|
||||
- **Concurrency** → `concurrency.md` — `context.Context` discipline, `errgroup`, `sync.OnceValue`, `goleak`, `-race`, channel selection rules.
|
||||
- **HTTP backend** → `backend-stack.md` — `gin` server skeleton, middleware ordering, SSE/streaming with `http.Flusher`, structured slog logging, graceful shutdown — distilled from the CLIProxyAPI codebase (a real proxy serving OpenAI/Gemini/Claude APIs).
|
||||
- **RPC** → `grpc-connect.md` — when to pick Connect vs grpc-go, codegen pipeline, protovalidate, streaming.
|
||||
- **DB** → `sqlc-pgx.md` — compile-time-safe SQL via sqlc + pgx connection pool + migrations via goose + testcontainers in CI.
|
||||
- **CLI** → `cobra-stack.md` — cobra layout, slog integration, graceful shutdown on signals, fang-style colored help.
|
||||
- **TUI** → `bubbletea-v2.md` — v2 model, `SetVirtualCursor(false)` + `tea.View{Cursor}` for CJK IME, why v1 was broken for Korean/Japanese/Chinese input.
|
||||
- **Testing** → `testing.md` — table-driven tests, `require` vs `assert`, `autogold` snapshots, `gopter` property tests, `testcontainers` for integration, `goleak` for goroutine leaks.
|
||||
- **Bootstrap** → `bootstrap.md` — `new-project.go` invocation, project layout (`cmd/`, `internal/`, `pkg/`), Taskfile, CI.
|
||||
- **Strict config** → `golangci-strict.md` — the canonical `.golangci.yml` with the full linter whitelist and per-linter rationale.
|
||||
- **One-liners** → `one-liners.md` — `go run` scripts with `//go:build ignore`, `gorun`-style invocation.
|
||||
|
||||
## The 250 pure LOC ceiling
|
||||
|
||||
Same rule as Python/Rust/TS: a `.go` file whose pure LOC (non-blank, non-comment) exceeds 250 is architecturally broken. Go encourages many small files in a single package, so this is *more* natural here than elsewhere — split by responsibility, keep one cohesive type and its methods per file.
|
||||
|
||||
The `cmd/server/main.go` is the most common violator. Refactor it: `main.go` only wires `os.Args` → `cmd.Execute()`. Anything else lives in `internal/`.
|
||||
|
||||
## Existing codebases — non-strict project
|
||||
|
||||
When editing an existing `.go` file that doesn't follow these rules: **write new code in strict style, don't refactor existing code in the same change.** Use the `remove-ai-slops` skill for branch-scope cleanup.
|
||||
|
||||
## Activation
|
||||
|
||||
This skill activates whenever you write or modify any `.go` file, `go.mod`, `go.sum`, `.golangci.yml`, `Taskfile.yml`, or any of the codegen specs (`*.proto`, `*.sql` next to `sqlc.yaml`, `openapi.yaml` next to `oapi-codegen.yaml`). Even one-off scripts get the strict treatment — that is what `//go:build ignore` + `go run` is for: production hygiene with throwaway ergonomics.
|
||||
|
||||
The references contain the recipes. **Read them before writing code. Re-read them when the model drifts.** The post-write architectural review loop is non-negotiable.
|
||||
@@ -0,0 +1,641 @@
|
||||
# HTTP Backend Stack — gin + slog + validator + pgx
|
||||
|
||||
The canonical production HTTP service skeleton. Distilled from the [CLIProxyAPI](https://github.com/router-for-me/CLIProxyAPI) codebase — a real proxy serving OpenAI / Gemini / Claude / Codex APIs in production, with SSE streaming, WebSocket upgrades, request logging, and hot-reload config.
|
||||
|
||||
If you are tempted to pick echo or chi instead, see `libraries.md` — gin wins on ecosystem, not technical merit, and the win is large enough to matter.
|
||||
|
||||
---
|
||||
|
||||
## `go.mod`
|
||||
|
||||
```go
|
||||
module github.com/your-org/myservice
|
||||
|
||||
go 1.23
|
||||
|
||||
require (
|
||||
github.com/gin-gonic/gin v1.10.1
|
||||
github.com/go-playground/validator/v10 v10.22.1
|
||||
github.com/caarlos0/env/v11 v11.2.2
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.7.6
|
||||
golang.org/x/sync v0.18.0
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Project structure
|
||||
|
||||
```
|
||||
cmd/server/main.go # ≤ 50 LOC; flags → run.Execute(ctx)
|
||||
internal/
|
||||
cmd/run.go # ~150 LOC; signal handling, config load, server.Run
|
||||
config/config.go # env-driven Config struct
|
||||
api/
|
||||
server.go # gin.Engine setup, route mounting, http.Server
|
||||
middleware/
|
||||
request_id.go
|
||||
request_logging.go
|
||||
auth.go
|
||||
recovery.go
|
||||
cors.go
|
||||
handlers/
|
||||
users.go # one file per resource
|
||||
streams.go # SSE / WebSocket endpoints
|
||||
domain/ # smart-constructor types (Email, UserID, ...)
|
||||
service/ # business logic
|
||||
store/ # pgx + sqlc
|
||||
obs/
|
||||
logger.go # slog setup
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## `cmd/server/main.go`
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/your-org/myservice/internal/cmd"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(),
|
||||
syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
if err := cmd.Execute(ctx); err != nil {
|
||||
slog.Error("fatal", slog.Any("err", err))
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
That is the entire `main`. Anything more is a smell.
|
||||
|
||||
---
|
||||
|
||||
## `internal/config/config.go`
|
||||
|
||||
```go
|
||||
package config
|
||||
|
||||
import (
|
||||
"time"
|
||||
"github.com/caarlos0/env/v11"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Host string `env:"HOST" envDefault:"0.0.0.0"`
|
||||
Port int `env:"PORT" envDefault:"8080"`
|
||||
DatabaseURL string `env:"DATABASE_URL,required"`
|
||||
ReadTimeout time.Duration `env:"READ_TIMEOUT" envDefault:"15s"`
|
||||
WriteTimeout time.Duration `env:"WRITE_TIMEOUT" envDefault:"30s"`
|
||||
ShutdownTimeout time.Duration `env:"SHUTDOWN_TIMEOUT" envDefault:"20s"`
|
||||
LogLevel string `env:"LOG_LEVEL" envDefault:"info"`
|
||||
LogFormat string `env:"LOG_FORMAT" envDefault:"json"`
|
||||
Env string `env:"ENV" envDefault:"development"`
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
var cfg Config
|
||||
if err := env.Parse(&cfg); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## `internal/obs/logger.go`
|
||||
|
||||
```go
|
||||
package obs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
)
|
||||
|
||||
type ctxKey struct{ name string }
|
||||
var requestIDKey = ctxKey{"request_id"}
|
||||
|
||||
func NewLogger(level, format string) *slog.Logger {
|
||||
var lvl slog.Level
|
||||
_ = lvl.UnmarshalText([]byte(level))
|
||||
|
||||
opts := &slog.HandlerOptions{Level: lvl, AddSource: true}
|
||||
|
||||
var h slog.Handler
|
||||
switch format {
|
||||
case "text":
|
||||
h = slog.NewTextHandler(os.Stdout, opts)
|
||||
default:
|
||||
h = slog.NewJSONHandler(os.Stdout, opts)
|
||||
}
|
||||
return slog.New(&ctxHandler{Handler: h})
|
||||
}
|
||||
|
||||
// ctxHandler pulls request_id from ctx into every log line.
|
||||
type ctxHandler struct{ slog.Handler }
|
||||
|
||||
func (h *ctxHandler) Handle(ctx context.Context, r slog.Record) error {
|
||||
if id, ok := ctx.Value(requestIDKey).(string); ok && id != "" {
|
||||
r.AddAttrs(slog.String("request_id", id))
|
||||
}
|
||||
return h.Handler.Handle(ctx, r)
|
||||
}
|
||||
|
||||
func WithRequestID(ctx context.Context, id string) context.Context {
|
||||
return context.WithValue(ctx, requestIDKey, id)
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## `internal/api/server.go`
|
||||
|
||||
```go
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/your-org/myservice/internal/api/handlers"
|
||||
"github.com/your-org/myservice/internal/api/middleware"
|
||||
"github.com/your-org/myservice/internal/config"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
cfg config.Config
|
||||
srv *http.Server
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func New(cfg config.Config, logger *slog.Logger, h *handlers.Handler) *Server {
|
||||
gin.SetMode(gin.ReleaseMode)
|
||||
r := gin.New()
|
||||
|
||||
// Middleware order matters — see "Middleware ordering" below.
|
||||
r.Use(
|
||||
middleware.RequestID(), // 1. assign request_id first
|
||||
middleware.Recovery(logger), // 2. recovery wraps everything
|
||||
middleware.RequestLogger(logger),
|
||||
middleware.CORS(),
|
||||
)
|
||||
|
||||
h.Mount(r)
|
||||
|
||||
return &Server{
|
||||
cfg: cfg,
|
||||
logger: logger,
|
||||
srv: &http.Server{
|
||||
Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port),
|
||||
Handler: r,
|
||||
ReadTimeout: cfg.ReadTimeout,
|
||||
WriteTimeout: cfg.WriteTimeout,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) Run(ctx context.Context) error {
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
s.logger.InfoContext(ctx, "server starting",
|
||||
slog.String("addr", s.srv.Addr))
|
||||
if err := s.srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
errCh <- err
|
||||
}
|
||||
close(errCh)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
s.logger.InfoContext(ctx, "shutdown signal received")
|
||||
shutdownCtx, cancel := context.WithTimeout(
|
||||
context.Background(), s.cfg.ShutdownTimeout)
|
||||
defer cancel()
|
||||
return s.srv.Shutdown(shutdownCtx)
|
||||
case err := <-errCh:
|
||||
return err
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- `gin.New()` not `gin.Default()` — `Default()` adds `Logger()` (text format, not slog) and `Recovery()` (no logger injection). We replace both.
|
||||
- `gin.SetMode(gin.ReleaseMode)` silences debug output. Production assumed.
|
||||
- `http.Server` with explicit timeouts. The default `nil` timeouts are a DoS waiting to happen.
|
||||
- Graceful shutdown: SIGINT/SIGTERM cancels the ctx → `Shutdown(shutdownCtx)` gives in-flight requests up to `ShutdownTimeout` to finish.
|
||||
|
||||
---
|
||||
|
||||
## Middleware ordering — the rule that actually matters
|
||||
|
||||
```
|
||||
RequestID → Recovery → Logger → CORS → Auth → Handler
|
||||
(1) (2) (3) (4) (5)
|
||||
```
|
||||
|
||||
1. **RequestID** is first so every subsequent middleware sees it.
|
||||
2. **Recovery** wraps everything after it. Order: a panic in CORS still gets caught.
|
||||
3. **Logger** sees the request_id and the recovered panic.
|
||||
4. **CORS** before Auth — OPTIONS preflight must return without auth.
|
||||
5. **Auth** is the last cross-cutting middleware. Per-route auth (admin-only) is mounted on a sub-router with extra middleware.
|
||||
|
||||
```go
|
||||
// Public routes — no auth
|
||||
api := r.Group("/api/v1")
|
||||
{
|
||||
api.POST("/auth/login", h.Login)
|
||||
api.GET("/healthz", h.Healthz)
|
||||
}
|
||||
|
||||
// Authenticated routes
|
||||
authed := r.Group("/api/v1", middleware.Auth(authSvc))
|
||||
{
|
||||
authed.GET("/users/:id", h.GetUser)
|
||||
authed.POST("/users", h.CreateUser)
|
||||
}
|
||||
|
||||
// Admin-only routes
|
||||
admin := r.Group("/api/v1/admin",
|
||||
middleware.Auth(authSvc),
|
||||
middleware.RequireRole("admin"))
|
||||
{
|
||||
admin.GET("/users", h.ListAllUsers)
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Middleware examples
|
||||
|
||||
### `middleware/request_id.go`
|
||||
|
||||
```go
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
"github.com/your-org/myservice/internal/obs"
|
||||
)
|
||||
|
||||
func RequestID() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
id := c.GetHeader("X-Request-ID")
|
||||
if id == "" {
|
||||
id = uuid.Must(uuid.NewV7()).String()
|
||||
}
|
||||
c.Request = c.Request.WithContext(obs.WithRequestID(c.Request.Context(), id))
|
||||
c.Header("X-Request-ID", id)
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### `middleware/recovery.go`
|
||||
|
||||
```go
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"runtime/debug"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func Recovery(logger *slog.Logger) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
logger.ErrorContext(c.Request.Context(), "panic recovered",
|
||||
slog.Any("panic", r),
|
||||
slog.String("stack", string(debug.Stack())),
|
||||
)
|
||||
if !c.Writer.Written() {
|
||||
c.JSON(http.StatusInternalServerError,
|
||||
gin.H{"error": "internal_error"})
|
||||
}
|
||||
c.Abort()
|
||||
}
|
||||
}()
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### `middleware/request_logging.go`
|
||||
|
||||
```go
|
||||
func RequestLogger(logger *slog.Logger) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
start := time.Now()
|
||||
c.Next()
|
||||
logger.InfoContext(c.Request.Context(), "http request",
|
||||
slog.String("method", c.Request.Method),
|
||||
slog.String("path", c.Request.URL.Path),
|
||||
slog.Int("status", c.Writer.Status()),
|
||||
slog.Int("bytes", c.Writer.Size()),
|
||||
slog.Duration("elapsed", time.Since(start)),
|
||||
slog.String("ip", c.ClientIP()),
|
||||
)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The `sloglint` linter enforces typed attrs (`slog.String(...)`) over `slog.Any("path", ...)`. Keep the form.
|
||||
|
||||
### `middleware/cors.go`
|
||||
|
||||
```go
|
||||
func CORS() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
c.Header("Access-Control-Allow-Origin", "*")
|
||||
c.Header("Access-Control-Allow-Methods", "GET, POST, PUT, PATCH, DELETE, OPTIONS")
|
||||
c.Header("Access-Control-Allow-Headers", "*")
|
||||
if c.Request.Method == http.MethodOptions {
|
||||
c.AbortWithStatus(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Note the explicit OPTIONS short-circuit — preflight must NOT go through Auth.
|
||||
|
||||
---
|
||||
|
||||
## Handlers — the canonical shape
|
||||
|
||||
```go
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/go-playground/validator/v10"
|
||||
"github.com/your-org/myservice/internal/domain"
|
||||
"github.com/your-org/myservice/internal/httperr"
|
||||
"github.com/your-org/myservice/internal/service"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
Users *service.UserService
|
||||
}
|
||||
|
||||
func (h *Handler) Mount(r gin.IRouter) {
|
||||
api := r.Group("/api/v1")
|
||||
api.POST("/users", h.CreateUser)
|
||||
api.GET("/users/:id", h.GetUser)
|
||||
}
|
||||
|
||||
type createUserReq struct {
|
||||
Email string `json:"email" binding:"required,email"`
|
||||
Username string `json:"username" binding:"required,alphanum,min=3,max=32"`
|
||||
}
|
||||
|
||||
func (h *Handler) CreateUser(c *gin.Context) {
|
||||
var req createUserReq
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
writeBindingError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
email, err := domain.NewEmail(req.Email)
|
||||
if err != nil {
|
||||
httperr.Write(c, err)
|
||||
return
|
||||
}
|
||||
username, err := domain.NewUsername(req.Username)
|
||||
if err != nil {
|
||||
httperr.Write(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
user, err := h.Users.Create(c.Request.Context(), email, username)
|
||||
if err != nil {
|
||||
httperr.Write(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusCreated, user)
|
||||
}
|
||||
|
||||
func writeBindingError(c *gin.Context, err error) {
|
||||
var vErr validator.ValidationErrors
|
||||
if errors.As(err, &vErr) {
|
||||
out := make(map[string]string, len(vErr))
|
||||
for _, fe := range vErr {
|
||||
out[fe.Field()] = fe.Tag()
|
||||
}
|
||||
c.JSON(http.StatusBadRequest, gin.H{"errors": out})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid_json"})
|
||||
}
|
||||
```
|
||||
|
||||
See `data-modeling.md` for the validator tag reference; see `error-handling.md` for the `httperr.Write` funnel.
|
||||
|
||||
---
|
||||
|
||||
## SSE streaming — the production pattern
|
||||
|
||||
CLIProxyAPI streams OpenAI-compatible SSE for hundreds of concurrent clients. The pattern:
|
||||
|
||||
```go
|
||||
func (h *Handler) StreamChat(c *gin.Context) {
|
||||
ctx, cancel := context.WithCancel(c.Request.Context())
|
||||
defer cancel()
|
||||
|
||||
// 1. Set SSE headers BEFORE writing any body
|
||||
c.Header("Content-Type", "text/event-stream")
|
||||
c.Header("Cache-Control", "no-cache")
|
||||
c.Header("Connection", "keep-alive")
|
||||
c.Header("X-Accel-Buffering", "no") // disable nginx buffering
|
||||
|
||||
// 2. Obtain the flusher — REQUIRED for streaming
|
||||
flusher, ok := c.Writer.(http.Flusher)
|
||||
if !ok {
|
||||
httperr.Write(c, errors.New("streaming unsupported"))
|
||||
return
|
||||
}
|
||||
|
||||
// 3. Pull chunks from upstream
|
||||
chunks, errs := h.svc.StreamCompletions(ctx, req)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return // client disconnected, ctx cancelled
|
||||
case chunk, ok := <-chunks:
|
||||
if !ok {
|
||||
fmt.Fprint(c.Writer, "data: [DONE]\n\n")
|
||||
flusher.Flush()
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(c.Writer, "data: %s\n\n", chunk)
|
||||
flusher.Flush()
|
||||
case err := <-errs:
|
||||
// Error mid-stream — emit as SSE event and bail
|
||||
fmt.Fprintf(c.Writer, "event: error\ndata: %s\n\n", err.Error())
|
||||
flusher.Flush()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Key facts:
|
||||
|
||||
- **Headers MUST be set before the first `Write`.** Otherwise gin auto-sets `Content-Type: text/plain`.
|
||||
- **`c.Writer.(http.Flusher)` is the streaming primitive.** Without `flusher.Flush()`, the response is buffered and arrives as one blob at the end.
|
||||
- **Always respond to `<-ctx.Done()`.** A disconnected client must stop upstream work — otherwise you generate tokens for nothing.
|
||||
- **The trailing `\n\n` per event is wire-mandatory** for SSE parsing. Missing it = the client never sees the event.
|
||||
|
||||
---
|
||||
|
||||
## WebSocket upgrade
|
||||
|
||||
```go
|
||||
import "github.com/gorilla/websocket" // still the canonical WS lib in 2026
|
||||
|
||||
var upgrader = websocket.Upgrader{
|
||||
ReadBufferSize: 4096,
|
||||
WriteBufferSize: 4096,
|
||||
CheckOrigin: func(r *http.Request) bool {
|
||||
// tighten in production
|
||||
return true
|
||||
},
|
||||
}
|
||||
|
||||
func (h *Handler) WebSocketEcho(c *gin.Context) {
|
||||
conn, err := upgrader.Upgrade(c.Writer, c.Request, nil)
|
||||
if err != nil {
|
||||
slog.ErrorContext(c.Request.Context(), "ws upgrade failed", slog.Any("err", err))
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
for {
|
||||
mt, msg, err := conn.ReadMessage()
|
||||
if err != nil { return }
|
||||
if err := conn.WriteMessage(mt, msg); err != nil { return }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
For long-lived connections, use `conn.SetReadDeadline` + `SetPongHandler` for keepalive. CLIProxyAPI's `wsrelay` package is a reference implementation.
|
||||
|
||||
---
|
||||
|
||||
## Database wiring — pgx pool, injected, never global
|
||||
|
||||
```go
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func NewPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) {
|
||||
cfg, err := pgxpool.ParseConfig(dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse dsn: %w", err)
|
||||
}
|
||||
cfg.MaxConns = 25
|
||||
cfg.MinConns = 5
|
||||
cfg.MaxConnLifetime = time.Hour
|
||||
cfg.MaxConnIdleTime = 30 * time.Minute
|
||||
|
||||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connect: %w", err)
|
||||
}
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("ping: %w", err)
|
||||
}
|
||||
return pool, nil
|
||||
}
|
||||
```
|
||||
|
||||
See `sqlc-pgx.md` for queries.
|
||||
|
||||
---
|
||||
|
||||
## Healthcheck
|
||||
|
||||
```go
|
||||
func (h *Handler) Healthz(c *gin.Context) {
|
||||
if err := h.pool.Ping(c.Request.Context()); err != nil {
|
||||
c.JSON(503, gin.H{"db": "down", "error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"ok": true})
|
||||
}
|
||||
```
|
||||
|
||||
Mount BEFORE auth. Health checks must be unauthenticated.
|
||||
|
||||
---
|
||||
|
||||
## Testing the server
|
||||
|
||||
```go
|
||||
func TestCreateUser_returns_201_for_valid_input(t *testing.T) {
|
||||
// Given
|
||||
h := newTestHandler(t)
|
||||
r := gin.New()
|
||||
h.Mount(r)
|
||||
|
||||
body := `{"email":"a@b.com","username":"alice"}`
|
||||
req := httptest.NewRequest("POST", "/api/v1/users", strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
// When
|
||||
r.ServeHTTP(rec, req)
|
||||
|
||||
// Then
|
||||
require.Equal(t, http.StatusCreated, rec.Code)
|
||||
var got struct{ ID string `json:"id"` }
|
||||
require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got))
|
||||
require.NotEmpty(t, got.ID)
|
||||
}
|
||||
```
|
||||
|
||||
See `testing.md` for full patterns (testcontainers integration, table-driven, goleak).
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- gin docs: https://gin-gonic.com/docs/
|
||||
- CLIProxyAPI (reference impl): https://github.com/router-for-me/CLIProxyAPI
|
||||
- pgx pool: https://pkg.go.dev/github.com/jackc/pgx/v5/pgxpool
|
||||
- SSE spec: https://html.spec.whatwg.org/multipage/server-sent-events.html
|
||||
- Go's `http.Server` graceful shutdown: https://pkg.go.dev/net/http#Server.Shutdown
|
||||
@@ -0,0 +1,328 @@
|
||||
# Bootstrap — Project Layout, Toolchain, Taskfile, CI
|
||||
|
||||
What every new Go project gets in the first 60 seconds. Drop the script in `scripts/go/new-project.go` does all of this — this document explains *what* it produces and *why*.
|
||||
|
||||
## Toolchain pin
|
||||
|
||||
`go.work` (monorepo) or just rely on `go.mod`'s `go 1.23` directive (single module). Go 1.21+ auto-downloads matching toolchain when the local `go` binary is older. **No `.tool-versions` / `asdf` / `mise` indirection required** unless your shop standardizes on it.
|
||||
|
||||
```bash
|
||||
# Confirm a working toolchain
|
||||
go env GOTOOLCHAIN # should be "auto" or your pinned version
|
||||
go version # ≥ 1.23
|
||||
```
|
||||
|
||||
## Required global installs
|
||||
|
||||
These are CLI tools, installed once per machine via `go install`:
|
||||
|
||||
```bash
|
||||
go install mvdan.cc/gofumpt@latest
|
||||
go install golang.org/x/tools/cmd/goimports@latest
|
||||
go install github.com/golangci/golangci-lint/cmd/golangci-lint@v2.0.0
|
||||
go install go.uber.org/nilaway/cmd/nilaway@latest
|
||||
go install github.com/sqlc-dev/sqlc/cmd/sqlc@latest
|
||||
go install github.com/pressly/goose/v3/cmd/goose@latest
|
||||
go install go.uber.org/mock/mockgen@latest
|
||||
go install github.com/go-task/task/v3/cmd/task@latest
|
||||
```
|
||||
|
||||
For Connect/protobuf projects, additionally:
|
||||
|
||||
```bash
|
||||
go install github.com/bufbuild/buf/cmd/buf@latest
|
||||
go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
|
||||
go install connectrpc.com/connect/cmd/protoc-gen-connect-go@latest
|
||||
```
|
||||
|
||||
## Project layout — the canonical tree
|
||||
|
||||
```
|
||||
myservice/
|
||||
├── go.mod
|
||||
├── go.sum
|
||||
├── Taskfile.yml # task runner
|
||||
├── .golangci.yml # see golangci-strict.md
|
||||
├── .editorconfig
|
||||
├── .gitignore
|
||||
├── README.md
|
||||
├── AGENTS.md # agent-readable project facts
|
||||
├── cmd/
|
||||
│ └── server/
|
||||
│ └── main.go # ONLY: parse flags, call cmd.Execute(); ≤ 50 LOC
|
||||
├── internal/ # NEVER importable from outside this module
|
||||
│ ├── api/ # transport layer (gin/connect routers)
|
||||
│ │ ├── server.go # gin engine setup, route registration
|
||||
│ │ ├── middleware/
|
||||
│ │ │ ├── request_id.go
|
||||
│ │ │ ├── logging.go
|
||||
│ │ │ └── auth.go
|
||||
│ │ └── handlers/
|
||||
│ │ ├── users.go
|
||||
│ │ └── users_test.go
|
||||
│ ├── domain/ # parse-don't-validate types, smart constructors
|
||||
│ │ ├── user.go
|
||||
│ │ └── email.go
|
||||
│ ├── service/ # business logic, depends on domain only
|
||||
│ │ └── user_service.go
|
||||
│ ├── store/ # persistence; sqlc-generated code lives here
|
||||
│ │ ├── sqlc/ # sqlc-generated, do not hand-edit
|
||||
│ │ ├── queries/ # *.sql files sqlc reads
|
||||
│ │ └── migrations/ # goose migrations
|
||||
│ ├── config/ # env-driven config (caarlos0/env)
|
||||
│ │ └── config.go
|
||||
│ └── obs/ # observability: slog setup, otel, healthz
|
||||
│ └── logger.go
|
||||
├── pkg/ # exportable libraries — only if you publish
|
||||
│ └── …
|
||||
├── proto/ # *.proto definitions (Connect/gRPC projects)
|
||||
│ └── service.proto
|
||||
├── gen/ # generated code (Connect, OpenAPI)
|
||||
│ └── service/v1/
|
||||
│ ├── service.pb.go
|
||||
│ └── servicev1connect/
|
||||
├── test/ # cross-cutting test helpers, fixtures
|
||||
└── .github/workflows/ci.yml
|
||||
```
|
||||
|
||||
**Rules**:
|
||||
|
||||
- `cmd/<binary>/main.go` is ≤ 50 LOC. Anything more lives in `internal/cmd/`.
|
||||
- `internal/` is **the** business code. Other modules cannot import it (Go compiler-enforced).
|
||||
- `pkg/` is for things you genuinely want third parties to import. Empty until proven otherwise.
|
||||
- No `utils/`, `helpers/`, `common/`, `shared/`. **REJECT.** Files are named after the concept they own.
|
||||
- One package per directory. One responsibility per package.
|
||||
|
||||
## `Taskfile.yml` — the entry point for every action
|
||||
|
||||
`go-task/task` is the modern Make replacement. Cross-platform, YAML, fast.
|
||||
|
||||
```yaml
|
||||
version: '3'
|
||||
|
||||
vars:
|
||||
BINARY: server
|
||||
PKG: ./cmd/server
|
||||
|
||||
tasks:
|
||||
default:
|
||||
deps: [fmt, lint, test]
|
||||
|
||||
fmt:
|
||||
desc: Format all Go files
|
||||
cmds:
|
||||
- gofumpt -w .
|
||||
- goimports -w -local "$(go list -m)" .
|
||||
|
||||
lint:
|
||||
desc: Run all linters
|
||||
cmds:
|
||||
- golangci-lint run --timeout 5m ./...
|
||||
- nilaway -include-pkgs "$(go list -m)/..." ./...
|
||||
|
||||
test:
|
||||
desc: Run tests with race detector
|
||||
cmds:
|
||||
- go test -race -shuffle=on -count=1 ./...
|
||||
|
||||
test-cover:
|
||||
desc: Coverage report
|
||||
cmds:
|
||||
- go test -race -shuffle=on -count=1 -coverprofile=coverage.out ./...
|
||||
- go tool cover -html=coverage.out -o coverage.html
|
||||
|
||||
build:
|
||||
desc: Build the binary
|
||||
cmds:
|
||||
- go build -trimpath -ldflags="-s -w" -o bin/{{.BINARY}} {{.PKG}}
|
||||
|
||||
run:
|
||||
desc: Run the server locally
|
||||
deps: [build]
|
||||
cmds:
|
||||
- ./bin/{{.BINARY}}
|
||||
|
||||
gen:
|
||||
desc: Run all code generators
|
||||
cmds:
|
||||
- task: gen:sqlc
|
||||
- task: gen:mocks
|
||||
- task: gen:proto
|
||||
|
||||
gen:sqlc:
|
||||
cmds:
|
||||
- sqlc generate
|
||||
sources:
|
||||
- internal/store/queries/*.sql
|
||||
- internal/store/sqlc.yaml
|
||||
generates:
|
||||
- internal/store/sqlc/*.go
|
||||
|
||||
gen:mocks:
|
||||
cmds:
|
||||
- go generate ./...
|
||||
|
||||
gen:proto:
|
||||
cmds:
|
||||
- buf generate
|
||||
sources:
|
||||
- proto/**/*.proto
|
||||
- buf.yaml
|
||||
- buf.gen.yaml
|
||||
|
||||
migrate:up:
|
||||
cmds:
|
||||
- goose -dir internal/store/migrations postgres "$DATABASE_URL" up
|
||||
|
||||
migrate:down:
|
||||
cmds:
|
||||
- goose -dir internal/store/migrations postgres "$DATABASE_URL" down
|
||||
|
||||
ci:
|
||||
desc: Everything CI does, locally
|
||||
deps: [fmt, lint, test, build]
|
||||
```
|
||||
|
||||
`task` (no args) runs format + lint + test in parallel where possible. `task ci` runs the full pipeline.
|
||||
|
||||
## `go.mod` template
|
||||
|
||||
```go
|
||||
module github.com/your-org/myservice
|
||||
|
||||
go 1.23
|
||||
|
||||
require (
|
||||
github.com/caarlos0/env/v11 v11.2.2
|
||||
github.com/gin-gonic/gin v1.10.1
|
||||
github.com/go-playground/validator/v10 v10.22.1
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.7.6
|
||||
golang.org/x/sync v0.18.0
|
||||
)
|
||||
```
|
||||
|
||||
Only direct deps listed; `go mod tidy` populates indirects.
|
||||
|
||||
## `.editorconfig`
|
||||
|
||||
```ini
|
||||
root = true
|
||||
|
||||
[*]
|
||||
indent_style = tab
|
||||
indent_size = 4
|
||||
end_of_line = lf
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
|
||||
[*.{yml,yaml,json,md}]
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
```
|
||||
|
||||
## `.gitignore`
|
||||
|
||||
```gitignore
|
||||
bin/
|
||||
coverage.out
|
||||
coverage.html
|
||||
*.test
|
||||
*.prof
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
|
||||
# Local env
|
||||
.env
|
||||
.env.local
|
||||
|
||||
# Secrets
|
||||
*.pem
|
||||
*.key
|
||||
```
|
||||
|
||||
## CI — minimal GitHub Actions
|
||||
|
||||
`.github/workflows/ci.yml`:
|
||||
|
||||
```yaml
|
||||
name: ci
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
ci:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.23'
|
||||
cache: true
|
||||
|
||||
- name: Install tools
|
||||
run: |
|
||||
go install mvdan.cc/gofumpt@latest
|
||||
go install github.com/golangci/golangci-lint/cmd/golangci-lint@v2.0.0
|
||||
go install go.uber.org/nilaway/cmd/nilaway@latest
|
||||
go install github.com/go-task/task/v3/cmd/task@latest
|
||||
|
||||
- name: Format check
|
||||
run: gofumpt -l . | (! grep .)
|
||||
|
||||
- name: Lint
|
||||
run: golangci-lint run --timeout 5m ./...
|
||||
|
||||
- name: Nilaway
|
||||
run: nilaway ./...
|
||||
|
||||
- name: Test
|
||||
run: go test -race -shuffle=on -count=1 ./...
|
||||
|
||||
- name: Build
|
||||
run: go build -trimpath ./...
|
||||
```
|
||||
|
||||
The order matters: format → lint → nilaway → test → build. Fail fast on the cheap checks.
|
||||
|
||||
## `AGENTS.md` — agent-readable project facts
|
||||
|
||||
Every new project gets an `AGENTS.md` at the root. The content is **machine-friendly**: short, declarative, no marketing prose. Example:
|
||||
|
||||
```markdown
|
||||
# AGENTS.md
|
||||
|
||||
Go 1.23+ HTTP service for {one-line purpose}.
|
||||
|
||||
## Commands
|
||||
- `task` — fmt + lint + test
|
||||
- `task build` — produce ./bin/server
|
||||
- `task gen` — regenerate sqlc + mocks + proto
|
||||
|
||||
## Architecture
|
||||
- `cmd/server/main.go` — entrypoint, ≤50 LOC
|
||||
- `internal/api/` — gin handlers + middleware
|
||||
- `internal/domain/` — smart-constructor types, no I/O
|
||||
- `internal/store/sqlc/` — generated; never hand-edit
|
||||
|
||||
## Conventions
|
||||
- `slog` for all logs; never `log.*`, never `fmt.Println`
|
||||
- `context.Context` first arg for every public function
|
||||
- Errors wrapped with `%w`; check with `errors.Is/As`
|
||||
- 250 pure LOC ceiling per file — split before adding lines
|
||||
```
|
||||
|
||||
The skill's `cmd/new-project.go` writes this file with project-specific values filled in.
|
||||
|
||||
## Sources
|
||||
|
||||
- Go modules reference: https://go.dev/ref/mod
|
||||
- go-task: https://taskfile.dev
|
||||
- golangci-lint v2: https://golangci-lint.run/docs/configuration/
|
||||
- Standard project layout debate: https://go.dev/doc/modules/layout (NOT `golang-standards/project-layout` — that repo is community, not official)
|
||||
@@ -0,0 +1,360 @@
|
||||
# Bubbletea v2 — TUI with First-Class CJK / IME Support
|
||||
|
||||
The TUI stack for 2026. Use **v2 RC**, not v1. If your users include Korean, Japanese, or Chinese speakers, v1 is broken — IME composition lands in the wrong cells. v2 fixes this. This document is the canonical setup.
|
||||
|
||||
The reference implementation this document is distilled from: [`code-yeongyu/bubbletea-wm`](https://github.com/code-yeongyu/bubbletea-wm) — a floating window manager built specifically to nail down v2 + IME.
|
||||
|
||||
---
|
||||
|
||||
## Why v2 (not v1) — the IME story
|
||||
|
||||
Bubbletea v1 manages cursor positioning in software ("virtual cursor"). It draws a `█` at the cursor position. The terminal's *real* cursor stays at `(0, 0)`.
|
||||
|
||||
This breaks every CJK input method. IME candidate windows (the popup showing Hangul composition choices for Korean, kana → kanji for Japanese, and pinyin lookup for Chinese) anchor to the terminal's **real** cursor position. With v1, the candidate window appears at top-left while you are typing somewhere in the middle of the screen.
|
||||
|
||||
Bubbletea v2 fixes this with two changes:
|
||||
|
||||
1. **`tea.View{Cursor: *tea.Cursor}`** — your `View()` method returns a view that *includes* the desired cursor position. The framework moves the terminal's real cursor there.
|
||||
2. **`textarea.SetVirtualCursor(false)`** — textareas no longer draw their own `█`. They expose `.Cursor()` so you can read where they want the real cursor.
|
||||
|
||||
Together: IME popups appear where the user is typing. As they should.
|
||||
|
||||
### Other v2 wins (incidental)
|
||||
|
||||
- `tea.MouseClickMsg` / `MouseMotionMsg` / `MouseReleaseMsg` instead of one coarse `MouseMsg`.
|
||||
- Cleaner `View` struct with `AltScreen`, `MouseMode` fields instead of `tea.Cmd` setters.
|
||||
- Pluggable rendering pipeline; better performance under high message volume.
|
||||
|
||||
---
|
||||
|
||||
## `go.mod`
|
||||
|
||||
```go
|
||||
module github.com/your-org/mytui
|
||||
|
||||
go 1.23
|
||||
|
||||
require (
|
||||
charm.land/bubbletea/v2 v2.0.0-rc.2
|
||||
charm.land/bubbles/v2 v2.0.0-rc.1
|
||||
charm.land/lipgloss/v2 v2.0.0-beta.3
|
||||
github.com/mattn/go-runewidth v0.0.19
|
||||
)
|
||||
```
|
||||
|
||||
The packages live under `charm.land/` (NOT `github.com/charmbracelet/...`) for v2. This is the Charm team's deliberate import-path break to keep v2 separate from v1 until stable.
|
||||
|
||||
---
|
||||
|
||||
## Minimal app — the IME-correct skeleton
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
tea "charm.land/bubbletea/v2"
|
||||
"charm.land/bubbles/v2/textarea"
|
||||
)
|
||||
|
||||
type model struct {
|
||||
width, height int
|
||||
ta textarea.Model
|
||||
}
|
||||
|
||||
func initial() model {
|
||||
ta := textarea.New()
|
||||
ta.Placeholder = "Type Korean / Japanese / Chinese here..."
|
||||
ta.SetWidth(60)
|
||||
ta.SetHeight(10)
|
||||
ta.SetVirtualCursor(false) // ← THE LINE. Without this, IME breaks.
|
||||
ta.Focus()
|
||||
return model{ta: ta}
|
||||
}
|
||||
|
||||
func (m model) Init() tea.Cmd { return textarea.Blink }
|
||||
|
||||
func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
switch msg := msg.(type) {
|
||||
case tea.WindowSizeMsg:
|
||||
m.width, m.height = msg.Width, msg.Height
|
||||
case tea.KeyPressMsg:
|
||||
if msg.String() == "ctrl+c" {
|
||||
return m, tea.Quit
|
||||
}
|
||||
}
|
||||
var cmd tea.Cmd
|
||||
m.ta, cmd = m.ta.Update(msg)
|
||||
return m, cmd
|
||||
}
|
||||
|
||||
func (m model) View() tea.View {
|
||||
var view tea.View
|
||||
view.AltScreen = true
|
||||
view.SetContent(m.ta.View())
|
||||
|
||||
// ── THE OTHER LINE. Position the REAL cursor for IME. ──
|
||||
if cursor := m.ta.Cursor(); cursor != nil {
|
||||
view.Cursor = cursor
|
||||
}
|
||||
return view
|
||||
}
|
||||
|
||||
func main() {
|
||||
if _, err := tea.NewProgram(initial(), tea.WithAltScreen()).Run(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Println("bye")
|
||||
}
|
||||
```
|
||||
|
||||
The two lines that matter:
|
||||
|
||||
1. `ta.SetVirtualCursor(false)` — disables the virtual `█`.
|
||||
2. `view.Cursor = cursor` (where `cursor = m.ta.Cursor()`) — exports the real cursor position to the framework.
|
||||
|
||||
Without **both**, IME breaks.
|
||||
|
||||
---
|
||||
|
||||
## CJK width — go-runewidth, not `len()`
|
||||
|
||||
Korean, Japanese, Chinese characters render as **two terminal cells** (wide characters per Unicode East Asian Width). Naive `len(string)` returns byte count, not display width. `utf8.RuneCountInString` returns rune count, also not display width.
|
||||
|
||||
Use `github.com/mattn/go-runewidth`:
|
||||
|
||||
```go
|
||||
import "github.com/mattn/go-runewidth"
|
||||
|
||||
func displayWidth(s string) int {
|
||||
return runewidth.StringWidth(s)
|
||||
}
|
||||
|
||||
// Wide character occupies two cells; pad accordingly
|
||||
for _, r := range s {
|
||||
cell := string(r)
|
||||
w := runewidth.RuneWidth(r)
|
||||
canvas = append(canvas, cell)
|
||||
if w == 2 {
|
||||
canvas = append(canvas, "") // placeholder for second cell
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`lipgloss/v2` uses `go-runewidth` internally — `lipgloss.Width("\u4e2d\u6587")` returns 4, not 2. **If you measure outside lipgloss, you must call runewidth directly.**
|
||||
|
||||
---
|
||||
|
||||
## Mouse — v2 has typed events
|
||||
|
||||
```go
|
||||
case tea.MouseClickMsg:
|
||||
// msg.X, msg.Y, msg.Button
|
||||
return m.handleClick(msg.X, msg.Y, msg.Button)
|
||||
|
||||
case tea.MouseMotionMsg:
|
||||
return m.handleHover(msg.X, msg.Y)
|
||||
|
||||
case tea.MouseReleaseMsg:
|
||||
return m.handleRelease(msg.X, msg.Y)
|
||||
```
|
||||
|
||||
Enable mouse via the `View`:
|
||||
|
||||
```go
|
||||
view.MouseMode = tea.MouseModeCellMotion // or MouseModeAll
|
||||
```
|
||||
|
||||
`CellMotion` reports clicks + motion-while-button-pressed (drag). `MouseModeAll` reports motion always — heavier, only when you need hover.
|
||||
|
||||
---
|
||||
|
||||
## Components from `bubbles/v2`
|
||||
|
||||
```go
|
||||
import (
|
||||
"charm.land/bubbles/v2/textarea"
|
||||
"charm.land/bubbles/v2/textinput"
|
||||
"charm.land/bubbles/v2/spinner"
|
||||
"charm.land/bubbles/v2/viewport"
|
||||
"charm.land/bubbles/v2/list"
|
||||
"charm.land/bubbles/v2/table"
|
||||
"charm.land/bubbles/v2/help"
|
||||
"charm.land/bubbles/v2/key"
|
||||
)
|
||||
```
|
||||
|
||||
All v2 components support `SetVirtualCursor(false)` where they accept text input. Use it for every text input that users might type CJK into — and "might" should be assumed *yes*.
|
||||
|
||||
---
|
||||
|
||||
## Styling — `lipgloss/v2`
|
||||
|
||||
```go
|
||||
import "charm.land/lipgloss/v2"
|
||||
|
||||
titleStyle := lipgloss.NewStyle().
|
||||
Bold(true).
|
||||
Foreground(lipgloss.Color("230")).
|
||||
Background(lipgloss.Color("62")).
|
||||
Padding(0, 1).
|
||||
Border(lipgloss.RoundedBorder()).
|
||||
BorderForeground(lipgloss.Color("63"))
|
||||
|
||||
rendered := titleStyle.Render("\u4e2d\u6587")
|
||||
```
|
||||
|
||||
`lipgloss/v2` width and padding correctly account for CJK display width. v1 did too — this is not a v2-specific fix, just a reminder.
|
||||
|
||||
---
|
||||
|
||||
## Architecture pattern — Model–Update–View
|
||||
|
||||
```
|
||||
+--------------------------------------------+
|
||||
| tea.Program runs the event loop |
|
||||
| |
|
||||
| loop: |
|
||||
| msg <- queue |
|
||||
| model, cmd = model.Update(msg) |
|
||||
| view = model.View() |
|
||||
| render(view) |
|
||||
| if cmd != nil: go run(cmd) -> queue |
|
||||
+--------------------------------------------+
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- **Model is a value type, not a pointer.** Bubbletea calls `Update` with a value receiver and expects a new value returned. Pointer receivers cause subtle bugs where state mutation leaks across draws.
|
||||
- **`Update` is pure.** No I/O. No goroutines started inline. Any I/O returns a `tea.Cmd` — Bubbletea runs it in a goroutine and feeds the result back as a message.
|
||||
- **`View` is read-only.** It returns a `tea.View` without modifying state.
|
||||
- **`tea.Cmd` is `func() tea.Msg`.** It runs once, returns a message, exits. For repeating work, use `tea.Tick` or a self-resending command.
|
||||
|
||||
```go
|
||||
// One-shot command
|
||||
func loadData() tea.Cmd {
|
||||
return func() tea.Msg {
|
||||
data, err := fetch()
|
||||
if err != nil { return errMsg{err} }
|
||||
return dataLoadedMsg{data}
|
||||
}
|
||||
}
|
||||
|
||||
// Periodic
|
||||
func tickEvery() tea.Cmd {
|
||||
return tea.Tick(time.Second, func(t time.Time) tea.Msg {
|
||||
return tickMsg{t}
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Splitting the model — sub-models
|
||||
|
||||
```go
|
||||
type model struct {
|
||||
list list.Model
|
||||
input textinput.Model
|
||||
spinner spinner.Model
|
||||
}
|
||||
|
||||
func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
var cmds []tea.Cmd
|
||||
var cmd tea.Cmd
|
||||
|
||||
m.list, cmd = m.list.Update(msg)
|
||||
cmds = append(cmds, cmd)
|
||||
|
||||
m.input, cmd = m.input.Update(msg)
|
||||
cmds = append(cmds, cmd)
|
||||
|
||||
m.spinner, cmd = m.spinner.Update(msg)
|
||||
cmds = append(cmds, cmd)
|
||||
|
||||
return m, tea.Batch(cmds...)
|
||||
}
|
||||
```
|
||||
|
||||
`tea.Batch` runs commands concurrently. The framework collects their results in the order they arrive.
|
||||
|
||||
When the model exceeds 250 LOC, split by sub-model into separate files:
|
||||
|
||||
```
|
||||
internal/ui/
|
||||
├── model.go # root model orchestration
|
||||
├── list.go # list sub-model state + update + view
|
||||
├── input.go # input sub-model
|
||||
└── spinner.go # spinner sub-model
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Testing TUI code — `teatest`
|
||||
|
||||
```go
|
||||
import "charm.land/bubbletea/v2/teatest"
|
||||
|
||||
func TestModel_typing_cjk_keeps_cursor_in_position(t *testing.T) {
|
||||
// Given
|
||||
m := initial()
|
||||
tm := teatest.NewTestModel(t, m, teatest.WithInitialTermSize(80, 24))
|
||||
|
||||
// When — simulate typing two CJK wide characters
|
||||
tm.Send(tea.KeyPressMsg{Code: '\u4e2d'})
|
||||
tm.Send(tea.KeyPressMsg{Code: '\u6587'})
|
||||
|
||||
// Then
|
||||
out := tm.FinalOutput(t)
|
||||
require.Contains(t, string(out), "\u4e2d\u6587")
|
||||
// Cursor should be at column 4 (two wide chars = 4 cells)
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
`teatest` lets you drive the model through synthetic messages and inspect the rendered output. Pair with `autogold` snapshots for full-view regression tests.
|
||||
|
||||
---
|
||||
|
||||
## Common antipatterns
|
||||
|
||||
| Bad | Why | Good |
|
||||
|---|---|---|
|
||||
| `tea.Program` with `tea.WithoutSignals()` | Ctrl-C does not work | Default signal handling |
|
||||
| Pointer receivers on Model | Bubbletea expects value semantics | Value receivers, return new model |
|
||||
| `time.Sleep` inside `Update` | Blocks the event loop | `tea.Tick` or async `tea.Cmd` |
|
||||
| `fmt.Println` for debug | Corrupts the rendered output | `tea.Printf` for logging, or write to a file |
|
||||
| `len(s)` for CJK width | Off by 2x | `runewidth.StringWidth(s)` |
|
||||
| `Bubbletea v1` for an app with text input | Korean/Japanese IME breaks | v2 + `SetVirtualCursor(false)` |
|
||||
| Drawing your own `█` block cursor in v2 | Conflicts with `view.Cursor` | Let the terminal handle it |
|
||||
|
||||
---
|
||||
|
||||
## Performance — when v2 starts to crawl
|
||||
|
||||
- **Reduce View frequency.** If the model changes 60 times/sec but the rendered view changes once/sec, gate redraws on a "dirty" flag.
|
||||
- **`viewport.Model` for scrollable content.** Avoid re-rendering thousands of lines on every keystroke.
|
||||
- **`Batch` your commands.** A series of synchronous `tea.Cmd` returns serializes; `tea.Batch` parallelizes.
|
||||
- **Profile with `tea.WithFPS(N)`** to cap repaint rate during development.
|
||||
|
||||
---
|
||||
|
||||
## When NOT to use Bubbletea
|
||||
|
||||
- The app is one prompt + one answer. Use `huh` (also from Charm) — simpler, no Model–Update–View ceremony.
|
||||
- The app is a long-running daemon with occasional status output. Use `slog` to stderr and `tea.Program` only if interactivity becomes necessary.
|
||||
- The app must run as a non-tty subprocess (CI, redirected stdin). `tea.Program` requires a tty for input. Detect via `term.IsTerminal(int(os.Stdin.Fd()))` and fall back to a non-interactive path.
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- bubbletea v2 RC: https://github.com/charmbracelet/bubbletea/tree/v2
|
||||
- bubbles v2: https://github.com/charmbracelet/bubbles/tree/v2
|
||||
- lipgloss v2: https://github.com/charmbracelet/lipgloss/tree/v2
|
||||
- bubbletea-wm (IME reference): https://github.com/code-yeongyu/bubbletea-wm
|
||||
- crush CLI (production IME impl): https://github.com/charmbracelet/crush
|
||||
- go-runewidth: https://github.com/mattn/go-runewidth
|
||||
- Unicode East Asian Width: https://www.unicode.org/reports/tr11/
|
||||
@@ -0,0 +1,468 @@
|
||||
# CLI Stack — cobra + slog + caarlos0/env + signal handling
|
||||
|
||||
The canonical Go CLI skeleton. `cobra` is the de facto framework — Kubernetes, Docker CLI, Helm, GitHub CLI, gh, Hugo all use it. Use it.
|
||||
|
||||
---
|
||||
|
||||
## Toolchain
|
||||
|
||||
```bash
|
||||
go install github.com/spf13/cobra-cli@latest
|
||||
cobra-cli init mytool
|
||||
cobra-cli add server
|
||||
cobra-cli add migrate
|
||||
```
|
||||
|
||||
`cobra-cli` scaffolds the `cmd/` package. Edit the result; do not regenerate.
|
||||
|
||||
---
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
mytool/
|
||||
├── go.mod
|
||||
├── main.go # ≤ 30 LOC, calls cmd.Execute
|
||||
├── cmd/
|
||||
│ ├── root.go # rootCmd, persistent flags, slog setup
|
||||
│ ├── server.go # `mytool server` subcommand
|
||||
│ ├── migrate.go # `mytool migrate` subcommand
|
||||
│ └── version.go # `mytool version` — auto-injected version
|
||||
├── internal/
|
||||
│ ├── config/
|
||||
│ └── server/
|
||||
└── Taskfile.yml
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## `main.go`
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/your-org/mytool/cmd"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(),
|
||||
syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
if err := cmd.Execute(ctx); err != nil {
|
||||
slog.Error("fatal", slog.Any("err", err))
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`signal.NotifyContext` (Go 1.16+) gives every subcommand a ctx that cancels on Ctrl-C. Subcommands plumb the ctx into their workers.
|
||||
|
||||
---
|
||||
|
||||
## `cmd/root.go`
|
||||
|
||||
```go
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
verbose bool
|
||||
logFormat string
|
||||
configPath string
|
||||
)
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "mytool",
|
||||
Short: "Short description of mytool",
|
||||
Long: `Long description, prose; cobra wraps it for --help.`,
|
||||
PersistentPreRunE: func(c *cobra.Command, args []string) error {
|
||||
return setupLogger()
|
||||
},
|
||||
SilenceUsage: true, // don't print --help on every error
|
||||
SilenceErrors: true, // we log them ourselves in Execute
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false,
|
||||
"enable debug logging")
|
||||
rootCmd.PersistentFlags().StringVar(&logFormat, "log-format", "text",
|
||||
"log format: text or json")
|
||||
rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "",
|
||||
"path to config file (optional)")
|
||||
}
|
||||
|
||||
func Execute(ctx context.Context) error {
|
||||
return rootCmd.ExecuteContext(ctx)
|
||||
}
|
||||
|
||||
func setupLogger() error {
|
||||
level := slog.LevelInfo
|
||||
if verbose { level = slog.LevelDebug }
|
||||
opts := &slog.HandlerOptions{Level: level}
|
||||
|
||||
var h slog.Handler
|
||||
switch logFormat {
|
||||
case "json":
|
||||
h = slog.NewJSONHandler(os.Stderr, opts)
|
||||
case "text":
|
||||
h = slog.NewTextHandler(os.Stderr, opts)
|
||||
default:
|
||||
return fmt.Errorf("invalid log-format %q", logFormat)
|
||||
}
|
||||
slog.SetDefault(slog.New(h))
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- `RunE` / `PersistentPreRunE` (the `E` variants) return errors. Use these; never use `Run` (no error return, encourages `log.Fatal`).
|
||||
- `SilenceUsage: true` + `SilenceErrors: true` together: cobra stops printing the full `--help` on every command failure (the default behavior is rude in production scripts).
|
||||
- `ExecuteContext` (cobra 1.8+) plumbs the ctx into every subcommand's `cmd.Context()`.
|
||||
|
||||
---
|
||||
|
||||
## `cmd/server.go`
|
||||
|
||||
```go
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/your-org/mytool/internal/server"
|
||||
)
|
||||
|
||||
var (
|
||||
serverAddr string
|
||||
)
|
||||
|
||||
var serverCmd = &cobra.Command{
|
||||
Use: "server",
|
||||
Short: "Run the HTTP server",
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
ctx := c.Context()
|
||||
slog.InfoContext(ctx, "starting", slog.String("addr", serverAddr))
|
||||
return server.Run(ctx, serverAddr)
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
serverCmd.Flags().StringVar(&serverAddr, "addr", ":8080",
|
||||
"listen address")
|
||||
rootCmd.AddCommand(serverCmd)
|
||||
}
|
||||
```
|
||||
|
||||
The subcommand is a thin shim — flags + log line + delegate to `internal/server`. Anything bigger violates the 250-LOC ceiling and belongs in `internal/`.
|
||||
|
||||
---
|
||||
|
||||
## Subcommands with arguments
|
||||
|
||||
```go
|
||||
var migrateUpCmd = &cobra.Command{
|
||||
Use: "up [N]",
|
||||
Short: "Apply N migrations (default: all)",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
n := -1 // all
|
||||
if len(args) == 1 {
|
||||
var err error
|
||||
n, err = strconv.Atoi(args[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid N: %w", err)
|
||||
}
|
||||
}
|
||||
return migrate.Up(c.Context(), n)
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
Use cobra's argument validators (`cobra.ExactArgs`, `cobra.MaximumNArgs`, `cobra.OnlyValidArgs`). They produce clean help text.
|
||||
|
||||
---
|
||||
|
||||
## Flag types — typed, not strings
|
||||
|
||||
```go
|
||||
// GOOD
|
||||
serverCmd.Flags().DurationVar(&timeout, "timeout", 30*time.Second, "request timeout")
|
||||
serverCmd.Flags().IntVar(&port, "port", 8080, "port")
|
||||
serverCmd.Flags().StringSliceVar(&hosts, "host", nil, "allowed hosts (repeatable)")
|
||||
|
||||
// BAD — manual parsing
|
||||
serverCmd.Flags().StringVar(&timeoutStr, "timeout", "30s", "")
|
||||
// ...then later: time.ParseDuration(timeoutStr)
|
||||
```
|
||||
|
||||
`pflag` (cobra's flag lib) has typed variants for every common type. Use them; the parsing and error messages are free.
|
||||
|
||||
---
|
||||
|
||||
## Bind flags to env vars
|
||||
|
||||
cobra + viper is overkill for env binding. Use `caarlos0/env/v11`:
|
||||
|
||||
```go
|
||||
type ServerOpts struct {
|
||||
Addr string `env:"ADDR" envDefault:":8080"`
|
||||
Timeout time.Duration `env:"TIMEOUT" envDefault:"30s"`
|
||||
}
|
||||
|
||||
var opts ServerOpts
|
||||
|
||||
var serverCmd = &cobra.Command{
|
||||
Use: "server",
|
||||
PersistentPreRunE: func(c *cobra.Command, args []string) error {
|
||||
// 1. Parse env first.
|
||||
if err := env.Parse(&opts); err != nil { return err }
|
||||
// 2. Flags override env if explicitly set.
|
||||
if c.Flags().Changed("addr") {
|
||||
opts.Addr, _ = c.Flags().GetString("addr")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
return server.Run(c.Context(), opts)
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
serverCmd.Flags().String("addr", "", "listen address (env: ADDR)")
|
||||
serverCmd.Flags().Duration("timeout", 0, "request timeout (env: TIMEOUT)")
|
||||
rootCmd.AddCommand(serverCmd)
|
||||
}
|
||||
```
|
||||
|
||||
Precedence: **flag (if set) > env > default**. Document the env var in the flag usage string.
|
||||
|
||||
---
|
||||
|
||||
## Version subcommand — build-injected
|
||||
|
||||
```go
|
||||
// cmd/version.go
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime/debug"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Set by -ldflags at build time, falls back to debug.BuildInfo.
|
||||
var (
|
||||
version = ""
|
||||
commit = ""
|
||||
date = ""
|
||||
)
|
||||
|
||||
var versionCmd = &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "Print version",
|
||||
Run: func(c *cobra.Command, args []string) {
|
||||
v, c2, d := resolveVersion()
|
||||
fmt.Printf("mytool %s (commit %s, built %s)\n", v, c2, d)
|
||||
},
|
||||
}
|
||||
|
||||
func resolveVersion() (string, string, string) {
|
||||
if version != "" { return version, commit, date }
|
||||
info, ok := debug.ReadBuildInfo()
|
||||
if !ok { return "dev", "unknown", "unknown" }
|
||||
|
||||
var vcs, hash, time string
|
||||
for _, s := range info.Settings {
|
||||
switch s.Key {
|
||||
case "vcs.revision": hash = s.Value
|
||||
case "vcs.time": time = s.Value
|
||||
case "vcs": vcs = s.Value
|
||||
}
|
||||
}
|
||||
return info.Main.Version, hash, time + " (" + vcs + ")"
|
||||
}
|
||||
|
||||
func init() { rootCmd.AddCommand(versionCmd) }
|
||||
```
|
||||
|
||||
Build with version injection:
|
||||
|
||||
```bash
|
||||
go build \
|
||||
-ldflags="-X 'github.com/your-org/mytool/cmd.version=v1.2.3' -X 'github.com/your-org/mytool/cmd.commit=$(git rev-parse --short HEAD)' -X 'github.com/your-org/mytool/cmd.date=$(date -u +%Y-%m-%dT%H:%M:%SZ)'" \
|
||||
-o bin/mytool ./
|
||||
```
|
||||
|
||||
The `debug.BuildInfo` fallback means a `go install`'d binary also has version info — no manual `-ldflags` needed.
|
||||
|
||||
---
|
||||
|
||||
## Shell completions
|
||||
|
||||
```go
|
||||
var completionCmd = &cobra.Command{
|
||||
Use: "completion [bash|zsh|fish|powershell]",
|
||||
Short: "Generate shell completion",
|
||||
Args: cobra.ExactValidArgs(1),
|
||||
ValidArgs: []string{"bash", "zsh", "fish", "powershell"},
|
||||
DisableFlagsInUseLine: true,
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
switch args[0] {
|
||||
case "bash": return rootCmd.GenBashCompletionV2(os.Stdout, true)
|
||||
case "zsh": return rootCmd.GenZshCompletion(os.Stdout)
|
||||
case "fish": return rootCmd.GenFishCompletion(os.Stdout, true)
|
||||
case "powershell": return rootCmd.GenPowerShellCompletion(os.Stdout)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
func init() { rootCmd.AddCommand(completionCmd) }
|
||||
```
|
||||
|
||||
User:
|
||||
|
||||
```bash
|
||||
mytool completion zsh > "${fpath[1]}/_mytool"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Interactive prompts — `huh` from charm
|
||||
|
||||
For prompts/forms (`Are you sure?`, "Pick an environment", multi-field forms):
|
||||
|
||||
```go
|
||||
import "github.com/charmbracelet/huh"
|
||||
|
||||
var confirm bool
|
||||
err := huh.NewConfirm().
|
||||
Title("Apply migrations to PRODUCTION?").
|
||||
Affirmative("Yes, do it").
|
||||
Negative("Abort").
|
||||
Value(&confirm).
|
||||
Run()
|
||||
```
|
||||
|
||||
`huh` replaces `survey` (which is no longer maintained). It composes with `lipgloss` for styling.
|
||||
|
||||
---
|
||||
|
||||
## Progress / spinners
|
||||
|
||||
```go
|
||||
import "github.com/charmbracelet/huh/spinner"
|
||||
|
||||
err := spinner.New().Title("Fetching...").Action(func() {
|
||||
// long-running work
|
||||
}).Run()
|
||||
```
|
||||
|
||||
For determinate progress (downloads, batch processing), use `vbauerster/mpb/v8`:
|
||||
|
||||
```go
|
||||
import "github.com/vbauerster/mpb/v8"
|
||||
|
||||
p := mpb.New(mpb.WithWidth(60))
|
||||
bar := p.AddBar(int64(total), /* decorators */)
|
||||
for i := 0; i < total; i++ {
|
||||
work()
|
||||
bar.Increment()
|
||||
}
|
||||
p.Wait()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Output — JSON vs text
|
||||
|
||||
Honor `--output json` for any CLI that scripts will parse:
|
||||
|
||||
```go
|
||||
var outputFmt string
|
||||
|
||||
rootCmd.PersistentFlags().StringVar(&outputFmt, "output", "text",
|
||||
"output format: text or json")
|
||||
|
||||
func render(v any) error {
|
||||
switch outputFmt {
|
||||
case "json":
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(v)
|
||||
case "text":
|
||||
return renderText(v)
|
||||
default:
|
||||
return fmt.Errorf("invalid --output %q", outputFmt)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The `text` format uses `lipgloss` tables or `aquasecurity/table` for nicely-aligned columns. The `json` format is for `jq`-style piping.
|
||||
|
||||
---
|
||||
|
||||
## Error semantics
|
||||
|
||||
- Return errors from `RunE`. Cobra catches them and the `Execute` wrapper logs + exits non-zero.
|
||||
- `os.Exit(1)` should appear **only in `main.go`**. Anywhere else means a subcommand cannot be tested.
|
||||
- For graceful early termination ("user cancelled"), return a sentinel and check it in `Execute`:
|
||||
```go
|
||||
var ErrCancelled = errors.New("cancelled by user")
|
||||
// ... return ErrCancelled
|
||||
// in main:
|
||||
if errors.Is(err, cmd.ErrCancelled) { os.Exit(130) } // 128 + SIGINT
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Testing CLI commands
|
||||
|
||||
```go
|
||||
func TestServerCmd_runs_with_default_addr(t *testing.T) {
|
||||
// Given
|
||||
buf := &bytes.Buffer{}
|
||||
rootCmd.SetOut(buf)
|
||||
rootCmd.SetErr(buf)
|
||||
rootCmd.SetArgs([]string{"server", "--addr", ":0"})
|
||||
|
||||
// When
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
err := rootCmd.ExecuteContext(ctx)
|
||||
|
||||
// Then
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, buf.String(), "starting")
|
||||
}
|
||||
```
|
||||
|
||||
`SetArgs` + `ExecuteContext` is the canonical pattern. Bind a ctx with a short deadline for tests that would otherwise block.
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- cobra docs: https://github.com/spf13/cobra/blob/main/site/content/user_guide.md
|
||||
- pflag: https://github.com/spf13/pflag
|
||||
- huh: https://github.com/charmbracelet/huh
|
||||
- caarlos0/env: https://github.com/caarlos0/env
|
||||
- signal.NotifyContext: https://pkg.go.dev/os/signal#NotifyContext
|
||||
@@ -0,0 +1,362 @@
|
||||
# Concurrency
|
||||
|
||||
Goroutines, context, errgroup, channels, locks, and the discipline that keeps them from leaking. Go makes concurrency *easy to start* and *easy to get wrong*. This document is the boring rule set.
|
||||
|
||||
---
|
||||
|
||||
## The four non-negotiables
|
||||
|
||||
1. **`ctx context.Context` is the first parameter of every public function that does I/O or can be cancelled.**
|
||||
2. **No goroutine without a shutdown path.** Every `go` keyword must answer "how does this stop?".
|
||||
3. **`-race` on every test run.** The `Taskfile.yml` and CI both enforce it.
|
||||
4. **`goleak` in `TestMain`** for every package that spawns goroutines. Catches leaks the race detector cannot.
|
||||
|
||||
---
|
||||
|
||||
## `context.Context` — the cancellation backbone
|
||||
|
||||
```go
|
||||
// GOOD — ctx as first param, propagated through
|
||||
func (s *UserService) Create(ctx context.Context, email Email) (User, error) {
|
||||
user, err := s.store.Insert(ctx, email)
|
||||
if err != nil {
|
||||
return User{}, fmt.Errorf("insert: %w", err)
|
||||
}
|
||||
if err := s.notifier.Welcome(ctx, user); err != nil {
|
||||
return User{}, fmt.Errorf("notify: %w", err)
|
||||
}
|
||||
return user, nil
|
||||
}
|
||||
|
||||
// BAD — creates a fresh ctx, breaks request cancellation
|
||||
func (s *UserService) Create(email Email) (User, error) {
|
||||
ctx := context.Background() // ← contextcheck linter rejects this
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
The `contextcheck` linter (enabled in `golangci-strict.md`) refuses any function that has `ctx context.Context` available but uses `context.Background()` instead.
|
||||
|
||||
### `context.Value` — use sparingly
|
||||
|
||||
```go
|
||||
// Typed key — never use a bare string
|
||||
type ctxKey struct{ name string }
|
||||
var requestIDKey = ctxKey{"request_id"}
|
||||
|
||||
func WithRequestID(ctx context.Context, id string) context.Context {
|
||||
return context.WithValue(ctx, requestIDKey, id)
|
||||
}
|
||||
|
||||
func RequestID(ctx context.Context) string {
|
||||
v, _ := ctx.Value(requestIDKey).(string)
|
||||
return v
|
||||
}
|
||||
```
|
||||
|
||||
**Rules**:
|
||||
- Keys are unexported struct types, not strings. Prevents collisions across packages.
|
||||
- `context.Value` is for *request-scoped metadata* (request ID, auth subject, trace span), NEVER for application-scoped dependencies.
|
||||
- Dependencies (loggers, DB pools, config) go in your service struct, not in `context.Value`.
|
||||
|
||||
### `WithTimeout` / `WithCancel` — always pair with `defer cancel()`
|
||||
|
||||
```go
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel() // ← MUST be deferred. fatcontext linter catches misses.
|
||||
|
||||
if err := slow(ctx); err != nil { ... }
|
||||
```
|
||||
|
||||
Forgetting `defer cancel()` leaks a context goroutine until the parent expires — the `lostcancel` vet check catches it.
|
||||
|
||||
---
|
||||
|
||||
## `errgroup` — the structured concurrency primitive
|
||||
|
||||
`golang.org/x/sync/errgroup` is Go's answer to Python's `asyncio.TaskGroup` or Rust's `JoinSet`. Use it instead of raw `go` for any group of related goroutines.
|
||||
|
||||
```go
|
||||
import "golang.org/x/sync/errgroup"
|
||||
|
||||
func FetchAll(ctx context.Context, urls []string) ([][]byte, error) {
|
||||
g, ctx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(8) // concurrency cap — leave unbounded = production outage
|
||||
|
||||
results := make([][]byte, len(urls))
|
||||
for i, u := range urls {
|
||||
g.Go(func() error {
|
||||
body, err := fetch(ctx, u)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch %s: %w", u, err)
|
||||
}
|
||||
results[i] = body
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err := g.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
```
|
||||
|
||||
Properties:
|
||||
|
||||
- `WithContext(parent)` returns a child ctx that gets cancelled on **first non-nil error**. All in-flight goroutines see `ctx.Done()` and bail.
|
||||
- `SetLimit(n)` blocks `g.Go(...)` when the in-flight count hits `n`. **Always set this.** Unbounded fan-out is how services die.
|
||||
- `g.Wait()` returns the **first** non-nil error. Others are dropped. If you need all errors, accumulate them manually:
|
||||
```go
|
||||
var mu sync.Mutex
|
||||
var errs []error
|
||||
// inside g.Go:
|
||||
// mu.Lock(); errs = append(errs, err); mu.Unlock()
|
||||
// after Wait, errors.Join(errs...)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Goroutine leaks — `goleak`
|
||||
|
||||
```go
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"go.uber.org/goleak"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
goleak.VerifyTestMain(m)
|
||||
}
|
||||
```
|
||||
|
||||
This single line at the top of `*_test.go` runs goleak's check after every test in the package. If a test leaks a goroutine, the run fails — pointing at which goroutine.
|
||||
|
||||
**The bug it catches**: starting a goroutine in `setUp` and never joining it. Common in DB connection pools, background workers, ticker loops. The race detector does NOT catch this.
|
||||
|
||||
If you have a known long-lived goroutine (a singleton background worker, a metrics exporter), use `goleak.IgnoreTopFunction`:
|
||||
|
||||
```go
|
||||
goleak.VerifyTestMain(m,
|
||||
goleak.IgnoreTopFunction("github.com/prometheus/client_golang/prometheus.(*Registry).Push"),
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Channels — the rules that hold
|
||||
|
||||
### Direction
|
||||
|
||||
```go
|
||||
// GOOD — direction in signatures
|
||||
func produce(out chan<- Item)
|
||||
func consume(in <-chan Item)
|
||||
func pipeline(in <-chan Item, out chan<- Item)
|
||||
```
|
||||
|
||||
Direction restricts misuse. A consumer cannot close the producer's channel.
|
||||
|
||||
### Closing
|
||||
|
||||
- **The sender closes.** Always. Never the receiver, never multiple senders.
|
||||
- **Multiple senders → use a `sync.WaitGroup` + one closer.**
|
||||
- **Closing a closed channel panics.** Closing a `nil` channel panics. Sending on a closed channel panics. Receiving from a closed channel returns zero value with `ok = false`.
|
||||
|
||||
```go
|
||||
// Canonical fan-in: multiple producers, one closer
|
||||
func fanIn(ctx context.Context, sources ...<-chan Item) <-chan Item {
|
||||
out := make(chan Item)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(len(sources))
|
||||
for _, src := range sources {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for item := range src {
|
||||
select {
|
||||
case out <- item:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
go func() { wg.Wait(); close(out) }()
|
||||
return out
|
||||
}
|
||||
```
|
||||
|
||||
### Selecting
|
||||
|
||||
```go
|
||||
select {
|
||||
case msg := <-incoming:
|
||||
handle(msg)
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(5 * time.Second):
|
||||
return ErrTimeout
|
||||
}
|
||||
```
|
||||
|
||||
- `time.After` allocates a timer each call — fine for occasional selects, **NOT for hot loops**. Use `time.NewTimer` + `timer.Reset` for repeat selects.
|
||||
- A `default:` case makes `select` non-blocking. Use deliberately, not by accident.
|
||||
|
||||
### Buffered vs unbuffered
|
||||
|
||||
- **Unbuffered** (`make(chan T)`) = synchronous handoff. Sender blocks until receiver is ready. Use for *coordination*.
|
||||
- **Buffered** (`make(chan T, n)`) = asynchronous up to `n`. Use for *decoupling producer rate from consumer rate*.
|
||||
|
||||
A buffered channel of size 1 acts as a **non-blocking signal**:
|
||||
|
||||
```go
|
||||
ready := make(chan struct{}, 1)
|
||||
// Producer
|
||||
select {
|
||||
case ready <- struct{}{}: // signal once, non-blocking
|
||||
default: // already signaled, skip
|
||||
}
|
||||
// Consumer
|
||||
<-ready
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Locks — the pyramid
|
||||
|
||||
```
|
||||
Highest level (preferred)
|
||||
channels (message passing — "share memory by communicating")
|
||||
errgroup / wait group
|
||||
|
||||
sync.RWMutex (many readers, occasional writer)
|
||||
sync.Mutex (mutual exclusion)
|
||||
|
||||
atomic.Int64 / atomic.Pointer (single-word lock-free)
|
||||
|
||||
Lowest level (rare)
|
||||
unsafe.Pointer + barriers (custom lock-free; needs -race AND review)
|
||||
```
|
||||
|
||||
### `sync.Mutex` — embed, don't expose
|
||||
|
||||
```go
|
||||
type Cache struct {
|
||||
mu sync.RWMutex
|
||||
items map[string]Entry
|
||||
}
|
||||
|
||||
func (c *Cache) Get(key string) (Entry, bool) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
e, ok := c.items[key]
|
||||
return e, ok
|
||||
}
|
||||
|
||||
func (c *Cache) Set(key string, e Entry) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.items[key] = e
|
||||
}
|
||||
```
|
||||
|
||||
- `sync.Mutex` is **not** copyable. The `copylocks` vet check catches `var c2 = c1` where `c1` has a mutex.
|
||||
- Always `defer mu.Unlock()` immediately after `Lock()`. Forgetting is the #1 deadlock cause.
|
||||
- Never call user code (callbacks, listener notifications) while holding the lock. Drop the lock, snapshot the data, release, then call out.
|
||||
|
||||
### `sync.OnceValue` / `sync.OnceFunc` (Go 1.21+)
|
||||
|
||||
Replacement for `sync.Once` for typed lazy init:
|
||||
|
||||
```go
|
||||
var loadConfig = sync.OnceValue(func() Config {
|
||||
var cfg Config
|
||||
if err := env.Parse(&cfg); err != nil { panic(err) }
|
||||
return cfg
|
||||
})
|
||||
|
||||
func handler() { cfg := loadConfig(); ... }
|
||||
```
|
||||
|
||||
Type-safe, no `sync.Once` + global variable boilerplate.
|
||||
|
||||
### Atomics — the typed API only
|
||||
|
||||
```go
|
||||
// Go 1.19+ — use the typed atomic.* family
|
||||
var counter atomic.Int64
|
||||
counter.Add(1)
|
||||
n := counter.Load()
|
||||
|
||||
// NEVER — the old function-style is type-unsafe
|
||||
atomic.AddInt64(&counter, 1) // ← rejected
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Time — inject a clock for testability
|
||||
|
||||
```go
|
||||
type Clock interface {
|
||||
Now() time.Time
|
||||
}
|
||||
|
||||
type realClock struct{}
|
||||
func (realClock) Now() time.Time { return time.Now() }
|
||||
|
||||
type Service struct {
|
||||
clock Clock
|
||||
}
|
||||
|
||||
// Tests
|
||||
import "github.com/benbjohnson/clock"
|
||||
fake := clock.NewMock()
|
||||
fake.Set(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||
svc := &Service{clock: fake}
|
||||
```
|
||||
|
||||
**Never call `time.Now()` in domain or service code.** The `time` package becomes a hidden dependency — tests become flaky, retries become time-of-day-dependent, expirations cannot be tested.
|
||||
|
||||
`time.Sleep` in production code is a code smell. Use:
|
||||
- `time.NewTicker` for periodic work (and a `<-ctx.Done()` exit).
|
||||
- `time.NewTimer` for one-shot delays.
|
||||
- `time.After` ONLY in select statements, ONLY in non-hot paths.
|
||||
|
||||
---
|
||||
|
||||
## Race detector — non-negotiable in CI
|
||||
|
||||
```bash
|
||||
go test -race -shuffle=on -count=1 ./...
|
||||
```
|
||||
|
||||
- `-race` instruments memory accesses; catches data races at runtime. ~10x slow-down — acceptable for tests, not production.
|
||||
- `-shuffle=on` randomizes test order; catches hidden ordering dependencies.
|
||||
- `-count=1` defeats the test cache. Without it, "passing" might mean "ran 3 weeks ago".
|
||||
|
||||
If a test ONLY fails under `-race`, the bug is real. Don't disable the test; fix the race.
|
||||
|
||||
---
|
||||
|
||||
## Common antipatterns
|
||||
|
||||
| Bad | Why | Good |
|
||||
|---|---|---|
|
||||
| `go func() { ... }()` with no `ctx` plumbing | Leaks on shutdown | `errgroup.WithContext` or pass ctx |
|
||||
| Bare `time.Sleep(d)` in production | Untestable, blocks | `time.NewTimer` + select with `ctx.Done()` |
|
||||
| Channel of `interface{}` | Loses type | Typed channel; use sealed interface if variants needed |
|
||||
| `sync.Mutex` in a struct passed by value | Locked copies, undefined behavior | Embed in pointer-receiver type; copylocks catches it |
|
||||
| Locking around an entire request handler | Serializes the whole API | Lock only the smallest critical section |
|
||||
| `for { select { ... } }` without `<-ctx.Done()` | Cannot stop | Add ctx case in every long-lived select |
|
||||
| `sync.WaitGroup.Add(1)` inside the goroutine | Race: Wait can return before Add | Add **before** `go` |
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- Go memory model: https://go.dev/ref/mem
|
||||
- `errgroup` package: https://pkg.go.dev/golang.org/x/sync/errgroup
|
||||
- `goleak`: https://github.com/uber-go/goleak
|
||||
- "Go concurrency patterns" (Pike): https://go.dev/blog/pipelines
|
||||
- Sync.OnceValue blog: https://go.dev/blog/synctest (1.24+ note: `testing/synctest` for time-controlled tests is now experimental)
|
||||
@@ -0,0 +1,329 @@
|
||||
# Data Modeling — Three Layers of Validation
|
||||
|
||||
Go has no Pydantic. Go has no Zod. **You do not need them**, but only if you wire three layers correctly. This document is the canonical pattern.
|
||||
|
||||
## The three layers
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ HTTP / RPC / CLI │
|
||||
│ Raw bytes, strings, untrusted input │
|
||||
│ │
|
||||
│ Layer 1: validator/v10 (struct tags) ◄── parse-once │
|
||||
│ OR protovalidate (proto) │
|
||||
│ │
|
||||
└──────────────────────────┬──────────────────────────────────┘
|
||||
│ raw req → domain.X
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Domain (internal/domain) │
|
||||
│ │
|
||||
│ Layer 2: Smart constructors + unexported fields │
|
||||
│ NewEmail(s) → (Email, error) │
|
||||
│ NewUserID(s) → (UserID, error) │
|
||||
│ │
|
||||
│ Once inside this layer, NO further validation. │
|
||||
│ The types prove correctness. │
|
||||
└──────────────────────────┬──────────────────────────────────┘
|
||||
│ domain.X (proven valid)
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Storage (internal/store) │
|
||||
│ │
|
||||
│ Layer 3: sqlc-generated row structs ↔ domain types │
|
||||
│ Hand-written mappers, NOT struct tags │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Each layer parses once, into the next layer's types. **A function in the domain layer should never receive a raw string and validate it.** If it does, the boundary above failed.
|
||||
|
||||
---
|
||||
|
||||
## Layer 1: HTTP boundary — `go-playground/validator/v10`
|
||||
|
||||
```go
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/go-playground/validator/v10"
|
||||
)
|
||||
|
||||
// CreateUserRequest is the wire format. Tags drive validation.
|
||||
type CreateUserRequest struct {
|
||||
Email string `json:"email" binding:"required,email"`
|
||||
Username string `json:"username" binding:"required,alphanum,min=3,max=32"`
|
||||
Age int `json:"age" binding:"required,gte=13,lte=130"`
|
||||
Country string `json:"country" binding:"required,iso3166_1_alpha2"`
|
||||
}
|
||||
|
||||
func (h *Handler) CreateUser(c *gin.Context) {
|
||||
var req CreateUserRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
// validator returns ValidationErrors with field-by-field detail
|
||||
var vErr validator.ValidationErrors
|
||||
if errors.As(err, &vErr) {
|
||||
c.JSON(400, gin.H{"errors": fieldErrors(vErr)})
|
||||
return
|
||||
}
|
||||
c.JSON(400, gin.H{"error": "invalid json"})
|
||||
return
|
||||
}
|
||||
|
||||
// Cross into domain — single point of failure
|
||||
email, err := domain.NewEmail(req.Email)
|
||||
if err != nil {
|
||||
c.JSON(400, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
username, err := domain.NewUsername(req.Username)
|
||||
if err != nil {
|
||||
c.JSON(400, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
user, err := h.svc.Create(c.Request.Context(), email, username, req.Age)
|
||||
if err != nil {
|
||||
h.writeServiceError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(201, user)
|
||||
}
|
||||
|
||||
func fieldErrors(vErr validator.ValidationErrors) map[string]string {
|
||||
out := make(map[string]string, len(vErr))
|
||||
for _, fe := range vErr {
|
||||
out[fe.Field()] = fe.Tag() + "(" + fe.Param() + ")"
|
||||
}
|
||||
return out
|
||||
}
|
||||
```
|
||||
|
||||
**Tag reference — the tags you actually use**:
|
||||
|
||||
| Tag | Meaning |
|
||||
|---|---|
|
||||
| `required` | Non-zero value |
|
||||
| `omitempty` (json) | Skip if zero |
|
||||
| `min=N` / `max=N` | Length (strings/slices) or value (numbers) |
|
||||
| `gte=N` / `lte=N` / `gt=N` / `lt=N` | Numeric comparison |
|
||||
| `email` | RFC 5322-ish email |
|
||||
| `url` | Valid URL |
|
||||
| `uuid` / `uuid4` / `uuid7` | UUID format |
|
||||
| `alphanum` / `alpha` / `numeric` | Character class |
|
||||
| `iso3166_1_alpha2` | Country code (US, KR, JP) |
|
||||
| `iso4217` | Currency code (USD, KRW) |
|
||||
| `oneof=a b c` | Enum of literal values |
|
||||
| `dive` | Apply rules to each element of slice/map |
|
||||
| `eqfield=Field` | Cross-field equality (e.g., password confirm) |
|
||||
|
||||
### Custom validators — register at startup
|
||||
|
||||
```go
|
||||
func init() {
|
||||
if v, ok := binding.Validator.Engine().(*validator.Validate); ok {
|
||||
_ = v.RegisterValidation("strongpassword", validateStrongPassword)
|
||||
}
|
||||
}
|
||||
|
||||
func validateStrongPassword(fl validator.FieldLevel) bool {
|
||||
s := fl.Field().String()
|
||||
return len(s) >= 12 && hasUpper(s) && hasDigit(s) && hasSymbol(s)
|
||||
}
|
||||
```
|
||||
|
||||
Use sparingly. Most domain rules belong in smart constructors, not validators.
|
||||
|
||||
---
|
||||
|
||||
## Layer 2: Domain — smart constructors
|
||||
|
||||
Covered in detail in `type-patterns.md`. Recap:
|
||||
|
||||
```go
|
||||
package domain
|
||||
|
||||
type Username struct{ raw string }
|
||||
|
||||
func NewUsername(s string) (Username, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) < 3 || len(s) > 32 {
|
||||
return Username{}, ErrInvalidUsername
|
||||
}
|
||||
if !isAlphanum(s) {
|
||||
return Username{}, ErrInvalidUsername
|
||||
}
|
||||
return Username{raw: s}, nil
|
||||
}
|
||||
|
||||
func (u Username) String() string { return u.raw }
|
||||
```
|
||||
|
||||
**Rule**: every domain type that has invariants has:
|
||||
|
||||
1. An unexported field holding the raw form.
|
||||
2. A `New<Type>(raw) (<Type>, error)` constructor as the sole entry point.
|
||||
3. A `String() string` for printing.
|
||||
4. `MarshalJSON` / `UnmarshalJSON` if it crosses a JSON boundary outside HTTP handlers (e.g., logging payloads, queue messages).
|
||||
5. Optionally: `Scan` and `Value` for `database/sql` interop (rare with sqlc).
|
||||
|
||||
---
|
||||
|
||||
## Layer 3: Storage — sqlc rows ↔ domain types
|
||||
|
||||
sqlc generates row structs from `.sql` files. **Do not put validation tags on them.** Map between sqlc rows and domain types explicitly:
|
||||
|
||||
```go
|
||||
// internal/store/user_store.go
|
||||
package store
|
||||
|
||||
import "myservice/internal/domain"
|
||||
|
||||
func (s *UserStore) Get(ctx context.Context, id domain.UserID) (domain.User, error) {
|
||||
row, err := s.q.GetUser(ctx, string(id))
|
||||
if err != nil {
|
||||
return domain.User{}, err
|
||||
}
|
||||
return rowToUser(row)
|
||||
}
|
||||
|
||||
func rowToUser(r sqlc.UserRow) (domain.User, error) {
|
||||
email, err := domain.NewEmail(r.Email)
|
||||
if err != nil {
|
||||
// DB invariant broken — this is a programmer error, not a user error
|
||||
return domain.User{}, fmt.Errorf("db invariant: invalid email for user %s: %w", r.ID, err)
|
||||
}
|
||||
username, err := domain.NewUsername(r.Username)
|
||||
if err != nil {
|
||||
return domain.User{}, fmt.Errorf("db invariant: invalid username: %w", err)
|
||||
}
|
||||
return domain.User{
|
||||
ID: domain.UserID(r.ID),
|
||||
Email: email,
|
||||
Username: username,
|
||||
Created: r.CreatedAt,
|
||||
}, nil
|
||||
}
|
||||
```
|
||||
|
||||
The mapping is verbose. **That is the point.** Each field is a deliberate choice; refactors flag every site.
|
||||
|
||||
---
|
||||
|
||||
## Discriminated unions (sum types) at the boundary
|
||||
|
||||
When a wire payload has variants (e.g., `{"type": "user.created", ...}` vs `{"type": "user.deleted", ...}`):
|
||||
|
||||
```go
|
||||
// Wire DTO with raw discriminator
|
||||
type EventDTO struct {
|
||||
Type string `json:"type" binding:"required,oneof=created deleted updated"`
|
||||
Payload json.RawMessage `json:"payload" binding:"required"`
|
||||
}
|
||||
|
||||
// Parse into the sealed domain type
|
||||
func ParseEvent(dto EventDTO) (event.Event, error) {
|
||||
switch dto.Type {
|
||||
case "created":
|
||||
var c event.Created
|
||||
if err := json.Unmarshal(dto.Payload, &c); err != nil {
|
||||
return nil, fmt.Errorf("decode created: %w", err)
|
||||
}
|
||||
return c, nil
|
||||
case "deleted":
|
||||
var d event.Deleted
|
||||
if err := json.Unmarshal(dto.Payload, &d); err != nil {
|
||||
return nil, fmt.Errorf("decode deleted: %w", err)
|
||||
}
|
||||
return d, nil
|
||||
case "updated":
|
||||
var u event.Updated
|
||||
if err := json.Unmarshal(dto.Payload, &u); err != nil {
|
||||
return nil, fmt.Errorf("decode updated: %w", err)
|
||||
}
|
||||
return u, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown event type %q", dto.Type)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The `exhaustive` linter on the switch + the `oneof` validation tag together cover both "unknown type" and "unhandled variant".
|
||||
|
||||
---
|
||||
|
||||
## Enums — typed string consts, not iota
|
||||
|
||||
```go
|
||||
// GOOD — string-based, JSON-serializes correctly, debuggable
|
||||
type Status string
|
||||
|
||||
const (
|
||||
StatusPending Status = "pending"
|
||||
StatusActive Status = "active"
|
||||
StatusClosed Status = "closed"
|
||||
)
|
||||
|
||||
func (s Status) IsValid() bool {
|
||||
switch s {
|
||||
case StatusPending, StatusActive, StatusClosed:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *Status) UnmarshalJSON(data []byte) error {
|
||||
var raw string
|
||||
if err := json.Unmarshal(data, &raw); err != nil { return err }
|
||||
parsed := Status(raw)
|
||||
if !parsed.IsValid() { return fmt.Errorf("invalid status %q", raw) }
|
||||
*s = parsed
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
**Never use `iota` enums for anything that crosses a wire boundary.** They serialize as integers, which (a) breaks debuggability, (b) makes reordering enum values a silent breaking change.
|
||||
|
||||
Use the validator tag `binding:"oneof=pending active closed"` to enforce at the HTTP boundary.
|
||||
|
||||
---
|
||||
|
||||
## Nullable fields — `*T` vs sentinel
|
||||
|
||||
Three choices, in order of preference:
|
||||
|
||||
1. **Sentinel zero value**: `Age int` with `0` meaning "unknown". Works when zero is genuinely unreachable as a valid value.
|
||||
2. **`sql.Null<T>`** for DB columns: `sql.NullString`, `sql.NullInt64`, `sql.NullTime`. sqlc generates these for nullable columns.
|
||||
3. **`*T`**: only when you need to distinguish "not provided" from "set to zero" in a JSON payload (PATCH semantics).
|
||||
|
||||
```go
|
||||
// PATCH payload — `*string` discriminates absent vs empty
|
||||
type UpdateUserRequest struct {
|
||||
Email *string `json:"email,omitempty"`
|
||||
Username *string `json:"username,omitempty"`
|
||||
}
|
||||
```
|
||||
|
||||
Avoid `*T` in domain types — it bloats every consumer with nil checks. Keep `*T` at the boundary, unwrap on the way in.
|
||||
|
||||
---
|
||||
|
||||
## Common AI-generated antipatterns this rejects
|
||||
|
||||
| Bad | Why | Good |
|
||||
|---|---|---|
|
||||
| `func handle(req map[string]any)` | No types, no validation | Define a struct, parse with `validator` |
|
||||
| `if email != "" { ... }` inside domain | Validation in the wrong layer | Make `email Email`, no check needed |
|
||||
| `type Status int` with `iota` for wire field | Silent breaking on reorder | `type Status string` with const literals |
|
||||
| Struct tags `json:"email,string"` (the `,string` coercion) | Magic coercion hides bad input | Strict parsing, fail-fast |
|
||||
| `json.Unmarshal` then range-check after | Two-step "validate after parse" | Use `validator` tags or custom `UnmarshalJSON` |
|
||||
| Reusing handler DTO as the domain type | Couples wire format to business logic | Two distinct types, explicit mapping |
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- go-playground/validator: https://github.com/go-playground/validator
|
||||
- gin binding internals: https://github.com/gin-gonic/gin/blob/master/binding/json.go
|
||||
- Parse, don't validate: https://lexi-lambda.github.io/blog/2019/11/05/parse-don-t-validate/
|
||||
- sqlc with custom types: https://docs.sqlc.dev/en/latest/howto/overrides.html
|
||||
@@ -0,0 +1,359 @@
|
||||
# Error Handling
|
||||
|
||||
Typed errors, wrap chains, `errors.Is` / `errors.As`, no panic in libraries, resource cleanup. Go errors look simple and are full of footguns. This document is the canonical set of moves.
|
||||
|
||||
---
|
||||
|
||||
## The five rules
|
||||
|
||||
1. **Every error is wrapped on the way up, with `%w`, with context.** Never `return err` from a non-trivial site.
|
||||
2. **Compare with `errors.Is`, not `==`.** Wrap chains break `==`. The `errorlint` linter forbids `==` on errors.
|
||||
3. **Cast with `errors.As`, not type assertion.** Same reason.
|
||||
4. **`panic` is reserved for programmer errors.** Library code never panics on user input or environment failures. Use `(T, error)`.
|
||||
5. **Resources released via `defer` immediately after acquisition.** No "I'll add it later".
|
||||
|
||||
---
|
||||
|
||||
## Sentinel errors — for invariant programmatic checks
|
||||
|
||||
```go
|
||||
package domain
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
ErrInvalidEmail = errors.New("domain: invalid email")
|
||||
ErrInvalidPhone = errors.New("domain: invalid phone")
|
||||
ErrInvalidAge = errors.New("domain: invalid age")
|
||||
)
|
||||
|
||||
func NewEmail(s string) (Email, error) {
|
||||
if !emailRe.MatchString(s) {
|
||||
return Email{}, fmt.Errorf("email %q: %w", s, ErrInvalidEmail)
|
||||
}
|
||||
return Email{raw: strings.ToLower(s)}, nil
|
||||
}
|
||||
```
|
||||
|
||||
Caller branches on identity:
|
||||
|
||||
```go
|
||||
email, err := domain.NewEmail(input)
|
||||
if errors.Is(err, domain.ErrInvalidEmail) {
|
||||
return c.JSON(400, gin.H{"error": "email format"})
|
||||
}
|
||||
```
|
||||
|
||||
`errors.Is` walks the wrap chain. `err == domain.ErrInvalidEmail` would have failed because `fmt.Errorf` wrapped it.
|
||||
|
||||
---
|
||||
|
||||
## Typed errors — when you need structured data
|
||||
|
||||
When callers need fields off the error (the offending value, the failing field name, the upstream HTTP status):
|
||||
|
||||
```go
|
||||
type ValidationError struct {
|
||||
Field string
|
||||
Value string
|
||||
Rule string
|
||||
}
|
||||
|
||||
func (e *ValidationError) Error() string {
|
||||
return fmt.Sprintf("validation: %s=%q failed %s", e.Field, e.Value, e.Rule)
|
||||
}
|
||||
|
||||
// Optional: identity sentinel for errors.Is comparisons
|
||||
var ErrValidation = errors.New("validation")
|
||||
|
||||
func (e *ValidationError) Is(target error) bool {
|
||||
return target == ErrValidation
|
||||
}
|
||||
```
|
||||
|
||||
Caller:
|
||||
|
||||
```go
|
||||
err := svc.Save(ctx, user)
|
||||
|
||||
var vErr *ValidationError
|
||||
if errors.As(err, &vErr) {
|
||||
// vErr.Field, vErr.Rule are available
|
||||
c.JSON(400, gin.H{"field": vErr.Field, "rule": vErr.Rule})
|
||||
return
|
||||
}
|
||||
```
|
||||
|
||||
**`errors.As` requires a non-nil pointer-to-pointer.** Almost always the type is `*ConcreteError`. Forgetting the leading `*` is the most common bug here.
|
||||
|
||||
---
|
||||
|
||||
## Wrapping — `%w` is mandatory
|
||||
|
||||
```go
|
||||
// BAD — drops context
|
||||
return err
|
||||
|
||||
// BAD — drops the error chain (errors.Is/As stops working)
|
||||
return fmt.Errorf("failed to save user: %v", err)
|
||||
|
||||
// GOOD — preserves chain via %w
|
||||
return fmt.Errorf("save user %s: %w", userID, err)
|
||||
```
|
||||
|
||||
The `errorlint` linter catches `%v` where `%w` was meant. **Wrap once per layer**, with the minimum useful context:
|
||||
|
||||
```
|
||||
api/handler: "create user request: %w"
|
||||
service: "validate inputs: %w"
|
||||
domain: "email %q: %w"
|
||||
```
|
||||
|
||||
Each frame adds one fact, not a duplicate. The top-level error message reads as a path: `create user request: validate inputs: email "foo": domain: invalid email`.
|
||||
|
||||
### `errors.Join` — multiple errors at once
|
||||
|
||||
```go
|
||||
// Validate all fields, collect all errors
|
||||
var errs []error
|
||||
if _, err := NewEmail(req.Email); err != nil {
|
||||
errs = append(errs, fmt.Errorf("email: %w", err))
|
||||
}
|
||||
if _, err := NewUsername(req.Username); err != nil {
|
||||
errs = append(errs, fmt.Errorf("username: %w", err))
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
```
|
||||
|
||||
`errors.Is` still walks each joined error. Use when reporting batch validation, not for "wrap two unrelated errors".
|
||||
|
||||
---
|
||||
|
||||
## Panics — when allowed, when banned
|
||||
|
||||
**Banned**:
|
||||
|
||||
- Anywhere a `(T, error)` could be returned.
|
||||
- Inside HTTP handlers (gin's `Recovery` middleware catches them, but you've already lost the error context).
|
||||
- Inside any goroutine that survives request lifetime.
|
||||
|
||||
**Allowed** (with documentation):
|
||||
|
||||
- Map literal init at package level: `var statusNames = map[Status]string{...}` followed by a `func init()` that panics if a const has no name. Catches the bug at startup, not runtime.
|
||||
- The `must*` convention for genuinely unrecoverable startup:
|
||||
```go
|
||||
func MustParseURL(s string) *url.URL {
|
||||
u, err := url.Parse(s)
|
||||
if err != nil { panic(err) }
|
||||
return u
|
||||
}
|
||||
// Use only with literals known at compile time:
|
||||
var defaultAPI = MustParseURL("https://api.example.com")
|
||||
```
|
||||
- `default:` case of an exhaustive sealed-interface switch — see `type-patterns.md`.
|
||||
|
||||
The `revive` linter rule `error-return` will flag suspect panic sites; treat them as bugs.
|
||||
|
||||
---
|
||||
|
||||
## `defer` for resources — the only safe pattern
|
||||
|
||||
```go
|
||||
func writeReport(path string) (err error) {
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create %s: %w", path, err)
|
||||
}
|
||||
defer func() {
|
||||
if cerr := f.Close(); cerr != nil && err == nil {
|
||||
err = fmt.Errorf("close %s: %w", path, cerr)
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := f.Write(data); err != nil {
|
||||
return fmt.Errorf("write %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
Key points:
|
||||
|
||||
- `defer f.Close()` immediately after `os.Create` — never further down.
|
||||
- Named return `(err error)` so the deferred close can mutate it on close failure.
|
||||
- `bodyclose` linter catches missed `defer resp.Body.Close()` for HTTP responses.
|
||||
- `sqlclosecheck` linter catches missed `defer rows.Close()` for SQL.
|
||||
|
||||
### `errors.Join` for multi-stage cleanup
|
||||
|
||||
```go
|
||||
func process(path string) (err error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil { return err }
|
||||
defer func() {
|
||||
err = errors.Join(err, f.Close())
|
||||
}()
|
||||
// ... use f ...
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
When both the main operation AND `Close` can fail, `errors.Join` reports both without dropping either.
|
||||
|
||||
---
|
||||
|
||||
## HTTP error responses — a single funnel
|
||||
|
||||
Build one helper, route all handler errors through it:
|
||||
|
||||
```go
|
||||
package httperr
|
||||
|
||||
type APIError struct {
|
||||
Status int `json:"-"`
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
func (e *APIError) Error() string { return e.Code + ": " + e.Message }
|
||||
|
||||
var (
|
||||
NotFound = &APIError{Status: 404, Code: "not_found", Message: "resource not found"}
|
||||
Unauthorized = &APIError{Status: 401, Code: "unauthorized", Message: "unauthorized"}
|
||||
BadRequest = &APIError{Status: 400, Code: "bad_request", Message: "bad request"}
|
||||
Internal = &APIError{Status: 500, Code: "internal", Message: "internal error"}
|
||||
)
|
||||
|
||||
// Wrap a domain error into an API error.
|
||||
func From(err error) *APIError {
|
||||
if err == nil { return nil }
|
||||
|
||||
var apiErr *APIError
|
||||
if errors.As(err, &apiErr) { return apiErr }
|
||||
|
||||
switch {
|
||||
case errors.Is(err, domain.ErrInvalidEmail),
|
||||
errors.Is(err, domain.ErrInvalidUsername):
|
||||
return &APIError{Status: 400, Code: "validation", Message: err.Error()}
|
||||
case errors.Is(err, ErrNotFound):
|
||||
return NotFound
|
||||
case errors.Is(err, ErrUnauthorized):
|
||||
return Unauthorized
|
||||
default:
|
||||
// unknown — log full chain, return generic
|
||||
slog.Error("unmapped error", slog.Any("err", err))
|
||||
return Internal
|
||||
}
|
||||
}
|
||||
|
||||
func Write(c *gin.Context, err error) {
|
||||
apiErr := From(err)
|
||||
c.JSON(apiErr.Status, apiErr)
|
||||
}
|
||||
```
|
||||
|
||||
Handlers become trivial:
|
||||
|
||||
```go
|
||||
func (h *Handler) Create(c *gin.Context) {
|
||||
user, err := h.svc.Create(c.Request.Context(), req)
|
||||
if err != nil {
|
||||
httperr.Write(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(201, user)
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## errgroup — error propagation across goroutines
|
||||
|
||||
```go
|
||||
import "golang.org/x/sync/errgroup"
|
||||
|
||||
func fetchAll(ctx context.Context, urls []string) ([][]byte, error) {
|
||||
g, ctx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(8) // concurrency cap
|
||||
|
||||
results := make([][]byte, len(urls))
|
||||
for i, u := range urls {
|
||||
g.Go(func() error {
|
||||
body, err := fetch(ctx, u)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch %s: %w", u, err)
|
||||
}
|
||||
results[i] = body
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
```
|
||||
|
||||
- `errgroup.WithContext` cancels remaining tasks on first error.
|
||||
- `SetLimit` bounds concurrency.
|
||||
- First non-nil error is returned; others are discarded — by design.
|
||||
|
||||
See `concurrency.md` for the full pattern.
|
||||
|
||||
---
|
||||
|
||||
## Logging errors — structured, once
|
||||
|
||||
```go
|
||||
slog.ErrorContext(ctx, "save user failed",
|
||||
slog.String("user_id", string(id)),
|
||||
slog.Any("err", err), // %w chain is fully rendered
|
||||
)
|
||||
```
|
||||
|
||||
**Log once, at the outermost frame.** Logging at every wrap site produces five log lines for one error.
|
||||
|
||||
The `sloglint` linter enforces `slog.Any("err", err)` over `slog.String("err", err.Error())` — the former preserves the chain when handlers walk the value.
|
||||
|
||||
---
|
||||
|
||||
## Antipatterns
|
||||
|
||||
| Bad | Why | Good |
|
||||
|---|---|---|
|
||||
| `_ = err` | Silent ignore | Handle, log, or wrap |
|
||||
| `if err != nil { return err }` chained 10 deep without wrap | No path info | Add one fact per layer: `fmt.Errorf("step: %w", err)` |
|
||||
| `panic(err)` in HTTP handlers | Loses error chain, hits gin Recovery | `httperr.Write(c, err)` |
|
||||
| `err.Error() == "some string"` | Brittle, breaks on wrap | Define a sentinel, use `errors.Is` |
|
||||
| `if err == sql.ErrNoRows` | Breaks under wrap | `errors.Is(err, sql.ErrNoRows)` |
|
||||
| `catch-all log.Fatal(err)` in library code | Crashes the caller's process | Return error, let main decide |
|
||||
| Returning a typed nil pointer wrapped in error interface | Classic "nil != nil" bug | Return explicit `nil` for the error |
|
||||
|
||||
The last bug deserves its own example:
|
||||
|
||||
```go
|
||||
// BUG — returns a non-nil error interface containing a nil concrete type
|
||||
func bad() error {
|
||||
var e *MyError = nil
|
||||
return e // interface wraps nil pointer; errors == nil is FALSE
|
||||
}
|
||||
|
||||
// Caller
|
||||
if err := bad(); err != nil {
|
||||
// ← entered, but err.(*MyError) is nil — surprise panic
|
||||
}
|
||||
```
|
||||
|
||||
Fix: return explicit `nil`, not a typed nil. The `nilnil` linter catches this in `(T, error)` returns.
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- Go blog "Working with Errors in Go 1.13+": https://go.dev/blog/go1.13-errors
|
||||
- `errors.Join` (Go 1.20+): https://pkg.go.dev/errors#Join
|
||||
- errorlint: https://github.com/polyfloyd/go-errorlint
|
||||
- nilaway nil-interface check: https://github.com/uber-go/nilaway
|
||||
@@ -0,0 +1,236 @@
|
||||
# Strict `.golangci.yml` (golangci-lint v2)
|
||||
|
||||
The single source of truth for "is this Go code acceptable". Drop this in unmodified. **Every linter below is enabled deliberately — read the rationale before disabling one.**
|
||||
|
||||
`golangci-lint` v2 changed config schema (top-level `version: "2"`). All v1 configs are incompatible. The block below is v2.
|
||||
|
||||
## `.golangci.yml`
|
||||
|
||||
```yaml
|
||||
version: "2"
|
||||
|
||||
run:
|
||||
timeout: 5m
|
||||
tests: true
|
||||
modules-download-mode: readonly
|
||||
|
||||
linters:
|
||||
default: none
|
||||
enable:
|
||||
# ── Correctness — bug catchers ───────────────────────────────
|
||||
- govet # stdlib vet, includes shadow, fieldalignment, nilness
|
||||
- staticcheck # SA1*-SA9* — the de facto Go correctness linter
|
||||
- errcheck # unhandled errors. ZERO tolerance.
|
||||
- errorlint # %w wrapping, errors.As vs type-assertion, errors.Is vs ==
|
||||
- nilerr # `return nil` after `err != nil` — classic bug
|
||||
- nilnil # returning `(nil, nil)` from a (*T, error) function
|
||||
- bodyclose # http.Response.Body not closed
|
||||
- rowserrcheck # sql.Rows.Err() not checked
|
||||
- sqlclosecheck # sql.Rows / sql.Stmt not closed
|
||||
- contextcheck # functions taking context.Context don't get context.Background()
|
||||
- fatcontext # context.WithValue() in a loop — leaks
|
||||
- copyloopvar # Go 1.22 loop-var capture — should now use the new semantics
|
||||
- intrange # use `for i := range N` (Go 1.22+) instead of `for i := 0; i < N; i++`
|
||||
- usetesting # use t.TempDir/t.Setenv over os.* in tests
|
||||
- testifylint # require vs assert correctness, ObjectsAreEqual misuse
|
||||
|
||||
# ── Style / readability — kept narrow to avoid bikeshedding ─
|
||||
- gofumpt # stricter gofmt
|
||||
- goimports # import grouping + local prefix
|
||||
- whitespace # leading/trailing whitespace
|
||||
- misspell # typos in comments and strings
|
||||
- unconvert # redundant type conversions
|
||||
- unparam # unused function parameters
|
||||
- ineffassign # ineffective assignments
|
||||
- dupword # duplicate words ("the the")
|
||||
|
||||
# ── Architecture — file size, complexity, dead code ─────────
|
||||
- gocognit # cognitive complexity per function (threshold 25)
|
||||
- gocyclo # cyclomatic complexity per function (threshold 15)
|
||||
- funlen # function length (90 lines, 60 statements)
|
||||
- lll # line length 120
|
||||
- nestif # excessive nesting depth (>4)
|
||||
- dupl # duplicate code blocks
|
||||
- revive # extensible replacement for golint; selected rules below
|
||||
- unused # unused vars/funcs/types
|
||||
|
||||
# ── Exhaustiveness — Go's weakest spot ──────────────────────
|
||||
- exhaustive # type switch and enum-like const groups completeness
|
||||
|
||||
# ── Security ────────────────────────────────────────────────
|
||||
- gosec # CWE-aware security scanner
|
||||
|
||||
# ── Logging ─────────────────────────────────────────────────
|
||||
- sloglint # slog attr style + no slog.Any(); enforce structured logs
|
||||
|
||||
# ── Performance ─────────────────────────────────────────────
|
||||
- perfsprint # fmt.Sprintf where strconv suffices
|
||||
- prealloc # slice prealloc when length is known
|
||||
- makezero # make([]T, n) with non-zero n then append (the classic bug)
|
||||
|
||||
linters-settings:
|
||||
errcheck:
|
||||
check-type-assertions: true
|
||||
check-blank: true # `_ = err` is a violation
|
||||
|
||||
govet:
|
||||
enable-all: true
|
||||
settings:
|
||||
shadow:
|
||||
strict: true
|
||||
fieldalignment:
|
||||
# On by default; this catches struct layouts wasting memory.
|
||||
# Disable per-file with //nolint:fieldalignment ONLY for boundary types
|
||||
# whose JSON tag order matters for OpenAPI doc stability.
|
||||
|
||||
errorlint:
|
||||
errorf: true # %w mandatory for wrapping
|
||||
asserts: true # errors.As over type-assertion on `error`
|
||||
comparison: true # errors.Is over ==
|
||||
|
||||
gocognit:
|
||||
min-complexity: 25
|
||||
|
||||
gocyclo:
|
||||
min-complexity: 15
|
||||
|
||||
funlen:
|
||||
lines: 90
|
||||
statements: 60
|
||||
ignore-comments: true
|
||||
|
||||
lll:
|
||||
line-length: 120
|
||||
tab-width: 4
|
||||
|
||||
nestif:
|
||||
min-complexity: 4
|
||||
|
||||
exhaustive:
|
||||
default-signifies-exhaustive: false
|
||||
check:
|
||||
- switch
|
||||
- map
|
||||
|
||||
gosec:
|
||||
excludes:
|
||||
- G104 # handled by errcheck/errorlint
|
||||
- G304 # file path provided as input — too noisy for CLIs
|
||||
|
||||
sloglint:
|
||||
no-mixed-args: true # all attr or all key-value, never mixed
|
||||
kv-only: false
|
||||
attr-only: true # force slog.String(...) form
|
||||
no-global: all # disallow slog.Info; force a logger receiver
|
||||
context: scope # require *Context variants where ctx is in scope
|
||||
static-msg: true # msg must be a string literal (not fmt.Sprintf)
|
||||
no-raw-keys: true # use slog.String("key", ...) not raw "key", "val"
|
||||
key-naming-case: snake
|
||||
|
||||
testifylint:
|
||||
enable-all: true
|
||||
disable:
|
||||
- require-error # We DO use assert.Error in table-driven loops
|
||||
|
||||
revive:
|
||||
severity: warning
|
||||
rules:
|
||||
- name: var-naming
|
||||
- name: package-comments
|
||||
- name: exported
|
||||
- name: error-return
|
||||
- name: error-naming
|
||||
- name: errorf # use fmt.Errorf instead of errors.New(fmt.Sprintf)
|
||||
- name: if-return
|
||||
- name: indent-error-flow
|
||||
- name: range-val-in-closure
|
||||
- name: redefines-builtin-id
|
||||
- name: superfluous-else
|
||||
- name: unhandled-error
|
||||
arguments:
|
||||
- "fmt.Print.*"
|
||||
- "fmt.Fprint.*"
|
||||
|
||||
perfsprint:
|
||||
integer-format: true
|
||||
error-format: true
|
||||
bool-format: true
|
||||
string-format: true
|
||||
|
||||
goimports:
|
||||
local-prefixes:
|
||||
- github.com/your-org
|
||||
|
||||
issues:
|
||||
max-issues-per-linter: 0
|
||||
max-same-issues: 0
|
||||
exclude-rules:
|
||||
# Tests get a longer leash on funlen + lll
|
||||
- path: _test\.go
|
||||
linters:
|
||||
- funlen
|
||||
- lll
|
||||
- dupl
|
||||
- gosec
|
||||
# Generated code never lints
|
||||
- path: \.pb\.go$
|
||||
linters: [all]
|
||||
- path: \.connect\.go$
|
||||
linters: [all]
|
||||
- path: ^.*sqlc/.*\.sql\.go$
|
||||
linters: [all]
|
||||
|
||||
formatters:
|
||||
enable:
|
||||
- gofumpt
|
||||
- goimports
|
||||
```
|
||||
|
||||
## Per-linter rationale (why each is on)
|
||||
|
||||
| Linter | What it catches | Why no compromise |
|
||||
|---|---|---|
|
||||
| `errcheck` (incl. `check-blank: true`) | `_ = err`, ignored errors from `Close()`, `Write()`, `json.Marshal()` | Silent error ignore is the #1 Go bug class. Banning `_ = err` forces a decision at every site. |
|
||||
| `errorlint` | `err == io.EOF` instead of `errors.Is(err, io.EOF)`; missing `%w` in `fmt.Errorf` | Once you wrap in middleware, `==` checks silently break. `errors.Is/As` is the only safe form. |
|
||||
| `nilerr` / `nilnil` | `return nil` after `err != nil`; `return nil, nil` from `(*T, error)` | Classic AI-generated bugs. Linter catches them mechanically. |
|
||||
| `bodyclose` | `defer resp.Body.Close()` missed | Single most common Go memory leak. |
|
||||
| `contextcheck` | `ctx := context.Background()` inside a function that received `ctx` | Breaks cancellation propagation — the entire reason ctx exists. |
|
||||
| `exhaustive` | `switch x.(type)` missing a sealed-interface variant | **Go's weakest type-system spot.** This linter is the closest thing to compiler-enforced exhaustiveness. |
|
||||
| `sloglint` | `slog.Info(...)` (global), mixed `Any`/typed attrs | Without this, structured logging silently degrades into string concatenation. |
|
||||
| `govet/shadow` strict | `err := ... ; if ... { err := ...; ... }` shadowing | Hides the real error from outer scope — extremely common. |
|
||||
| `govet/fieldalignment` | Struct field order wasting memory | Cheap correctness signal. Disable per-file when JSON tag order matters for OpenAPI. |
|
||||
| `copyloopvar` + `intrange` | Pre-1.22 loop-var capture and old `for i := 0; i < N; i++` | The language modernized; the lint enforces it. |
|
||||
| `usetesting` | `os.Setenv` / `os.Mkdir` in tests instead of `t.Setenv` / `t.TempDir` | Avoids test isolation bugs. |
|
||||
| `gocognit` / `gocyclo` / `funlen` | Functions exceeding cognitive thresholds | Direct architectural signal — same purpose as the 250 LOC ceiling, at function granularity. |
|
||||
| `gosec` | CWE patterns — SQL injection, weak crypto, path traversal | Production must pass this. |
|
||||
| `testifylint` | `assert.Equal` where `require.Equal` was meant; `ObjectsAreEqual` misuse | Subtle test-correctness bugs. |
|
||||
| `perfsprint` | `fmt.Sprintf("%d", n)` instead of `strconv.Itoa(n)` | 5–10x faster in tight loops, lints catch the lazy form. |
|
||||
|
||||
## `nolint` policy
|
||||
|
||||
`//nolint:linter1,linter2 // <reason>` is permitted with **two hard rules**:
|
||||
|
||||
1. **One linter at a time per directive.** No `//nolint:all`. No omitting the linter name.
|
||||
2. **A reason after `//` is mandatory.** "Generated code", "false positive — protobuf imports", "OpenAPI field order" are acceptable. "Ignore" is not.
|
||||
|
||||
The skill auto-rejects `//nolint` without a reason. So does `revive` if you enable its `nolint` rule.
|
||||
|
||||
## CI gate
|
||||
|
||||
```bash
|
||||
gofumpt -l . | (! grep .) # format
|
||||
golangci-lint run --timeout 5m ./... # everything above
|
||||
go vet -vettool=$(which fieldalignment) ./... # extra check (also in govet)
|
||||
nilaway ./... # nil-deref static analysis
|
||||
go test -race -shuffle=on -count=1 ./... # races + ordering
|
||||
```
|
||||
|
||||
Any non-zero exit = the change does not ship.
|
||||
|
||||
## Sources
|
||||
|
||||
- golangci-lint v2 docs: https://golangci-lint.run/docs/configuration/
|
||||
- staticcheck rules: https://staticcheck.dev/docs/checks
|
||||
- sloglint: https://github.com/go-simpler/sloglint
|
||||
- exhaustive: https://github.com/nishanths/exhaustive
|
||||
- nilaway: https://github.com/uber-go/nilaway
|
||||
@@ -0,0 +1,375 @@
|
||||
# RPC — Connect-Go (default) + grpc-go (fallback) + protovalidate
|
||||
|
||||
`connectrpc/connect-go` is the default. It is wire-compatible with gRPC, also speaks Connect protocol + gRPC-Web from browsers, and uses ordinary `net/http` so middleware (logging, auth, tracing) composes the same way as REST. Reach for raw `grpc-go` only when you need a gRPC-specific feature Connect lacks.
|
||||
|
||||
---
|
||||
|
||||
## When Connect vs grpc-go
|
||||
|
||||
| Need | Use |
|
||||
|---|---|
|
||||
| Standard unary + server-streaming + client-streaming | **Connect** |
|
||||
| Browser client without `grpc-web` proxy | **Connect** (native gRPC-Web support) |
|
||||
| HTTP/1.1 fallback for hostile networks | **Connect** (gRPC requires HTTP/2 end-to-end) |
|
||||
| Server reflection for `grpcurl` | grpc-go (Connect has reflection too, but ecosystem smaller) |
|
||||
| Bidirectional streaming with frame-level control | grpc-go |
|
||||
| Strict gRPC environment (Envoy with gRPC filters, Istio strict mode) | grpc-go |
|
||||
|
||||
**Default**: Connect. The default has been correct since 2024.
|
||||
|
||||
---
|
||||
|
||||
## Toolchain — Buf, not protoc
|
||||
|
||||
```bash
|
||||
go install github.com/bufbuild/buf/cmd/buf@latest
|
||||
go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
|
||||
go install connectrpc.com/connect/cmd/protoc-gen-connect-go@latest
|
||||
go install github.com/bufbuild/protovalidate/cmd/protoc-gen-go-vtproto@latest
|
||||
```
|
||||
|
||||
Buf replaces `protoc` for everything: linting, breaking-change detection, codegen, formatting. The `protoc` toolchain is dead-letter walking — every modern proto project uses Buf.
|
||||
|
||||
---
|
||||
|
||||
## Project layout
|
||||
|
||||
```
|
||||
proto/
|
||||
buf.yaml
|
||||
buf.gen.yaml
|
||||
buf.lock
|
||||
myservice/v1/
|
||||
user.proto
|
||||
auth.proto
|
||||
|
||||
gen/
|
||||
myservice/v1/
|
||||
user.pb.go # protoc-gen-go output
|
||||
auth.pb.go
|
||||
myservicev1connect/ # protoc-gen-connect-go output
|
||||
user.connect.go
|
||||
auth.connect.go
|
||||
```
|
||||
|
||||
**`gen/` is committed.** Generated code is part of the API contract; CI proves it is up-to-date.
|
||||
|
||||
---
|
||||
|
||||
## `buf.yaml`
|
||||
|
||||
```yaml
|
||||
version: v2
|
||||
modules:
|
||||
- path: proto
|
||||
lint:
|
||||
use:
|
||||
- STANDARD
|
||||
breaking:
|
||||
use:
|
||||
- FILE
|
||||
```
|
||||
|
||||
## `buf.gen.yaml`
|
||||
|
||||
```yaml
|
||||
version: v2
|
||||
managed:
|
||||
enabled: true
|
||||
override:
|
||||
- file_option: go_package_prefix
|
||||
value: github.com/your-org/myservice/gen
|
||||
plugins:
|
||||
- remote: buf.build/protocolbuffers/go
|
||||
out: gen
|
||||
opt:
|
||||
- paths=source_relative
|
||||
- remote: buf.build/connectrpc/go
|
||||
out: gen
|
||||
opt:
|
||||
- paths=source_relative
|
||||
- remote: buf.build/bufbuild/validate-go
|
||||
out: gen
|
||||
opt:
|
||||
- paths=source_relative
|
||||
```
|
||||
|
||||
The `buf.build/...` plugin URIs use Buf's hosted remote registry — no local plugin installation needed.
|
||||
|
||||
## Taskfile target
|
||||
|
||||
```yaml
|
||||
gen:proto:
|
||||
cmds:
|
||||
- buf lint
|
||||
- buf format -w
|
||||
- buf generate
|
||||
sources:
|
||||
- proto/**/*.proto
|
||||
- buf.yaml
|
||||
- buf.gen.yaml
|
||||
```
|
||||
|
||||
Run `task gen:proto` after editing any `.proto`. CI runs `buf generate` then `git diff --exit-code` to catch stale generated code.
|
||||
|
||||
---
|
||||
|
||||
## A `.proto` with validation
|
||||
|
||||
```proto
|
||||
syntax = "proto3";
|
||||
|
||||
package myservice.v1;
|
||||
|
||||
import "buf/validate/validate.proto";
|
||||
|
||||
option go_package = "github.com/your-org/myservice/gen/myservice/v1;myservicev1";
|
||||
|
||||
service UserService {
|
||||
rpc CreateUser(CreateUserRequest) returns (CreateUserResponse);
|
||||
rpc GetUser(GetUserRequest) returns (GetUserResponse);
|
||||
rpc StreamEvents(StreamEventsRequest) returns (stream Event);
|
||||
}
|
||||
|
||||
message CreateUserRequest {
|
||||
string email = 1 [(buf.validate.field).string.email = true];
|
||||
string username = 2 [
|
||||
(buf.validate.field).string.min_len = 3,
|
||||
(buf.validate.field).string.max_len = 32,
|
||||
(buf.validate.field).string.pattern = "^[a-zA-Z0-9_]+$"
|
||||
];
|
||||
int32 age = 3 [
|
||||
(buf.validate.field).int32.gte = 13,
|
||||
(buf.validate.field).int32.lte = 130
|
||||
];
|
||||
}
|
||||
|
||||
message CreateUserResponse {
|
||||
User user = 1;
|
||||
}
|
||||
|
||||
message User {
|
||||
string id = 1;
|
||||
string email = 2;
|
||||
string username = 3;
|
||||
google.protobuf.Timestamp created_at = 4;
|
||||
}
|
||||
```
|
||||
|
||||
`protovalidate` replaces the abandoned `protoc-gen-validate` — it is the official Buf-backed successor as of 2024, supported by Connect's interceptor pipeline.
|
||||
|
||||
---
|
||||
|
||||
## Server — Connect
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"connectrpc.com/connect"
|
||||
"buf.build/go/protovalidate"
|
||||
validateinterceptor "connectrpc.com/validate"
|
||||
"golang.org/x/net/http2"
|
||||
"golang.org/x/net/http2/h2c"
|
||||
|
||||
myservicev1 "github.com/your-org/myservice/gen/myservice/v1"
|
||||
"github.com/your-org/myservice/gen/myservice/v1/myservicev1connect"
|
||||
)
|
||||
|
||||
type UserServer struct {
|
||||
svc *UserService
|
||||
}
|
||||
|
||||
func (s *UserServer) CreateUser(
|
||||
ctx context.Context,
|
||||
req *connect.Request[myservicev1.CreateUserRequest],
|
||||
) (*connect.Response[myservicev1.CreateUserResponse], error) {
|
||||
|
||||
// protovalidate already ran via the interceptor below.
|
||||
// req.Msg is guaranteed to satisfy the .proto constraints.
|
||||
|
||||
user, err := s.svc.Create(ctx, req.Msg.Email, req.Msg.Username, req.Msg.Age)
|
||||
if err != nil {
|
||||
return nil, mapError(err)
|
||||
}
|
||||
return connect.NewResponse(&myservicev1.CreateUserResponse{
|
||||
User: userToProto(user),
|
||||
}), nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
validator, _ := protovalidate.New()
|
||||
interceptors := connect.WithInterceptors(
|
||||
loggingInterceptor(),
|
||||
validateinterceptor.NewInterceptor(validator),
|
||||
)
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle(myservicev1connect.NewUserServiceHandler(
|
||||
&UserServer{svc: newUserService()},
|
||||
interceptors,
|
||||
))
|
||||
|
||||
// h2c lets the server speak HTTP/2 cleartext for gRPC clients.
|
||||
srv := &http.Server{
|
||||
Addr: ":8080",
|
||||
Handler: h2c.NewHandler(mux, &http2.Server{}),
|
||||
}
|
||||
slog.Info("rpc server listening", slog.String("addr", srv.Addr))
|
||||
if err := srv.ListenAndServe(); err != nil { slog.Error("rpc", slog.Any("err", err)) }
|
||||
}
|
||||
```
|
||||
|
||||
The handler is **just an `http.Handler`** — mount it in the same `http.ServeMux` as your REST routes if you want one binary serving both.
|
||||
|
||||
---
|
||||
|
||||
## Error mapping — Connect codes
|
||||
|
||||
```go
|
||||
func mapError(err error) error {
|
||||
if err == nil { return nil }
|
||||
|
||||
switch {
|
||||
case errors.Is(err, domain.ErrInvalidEmail),
|
||||
errors.Is(err, domain.ErrInvalidUsername):
|
||||
return connect.NewError(connect.CodeInvalidArgument, err)
|
||||
case errors.Is(err, ErrNotFound):
|
||||
return connect.NewError(connect.CodeNotFound, err)
|
||||
case errors.Is(err, ErrUnauthorized):
|
||||
return connect.NewError(connect.CodeUnauthenticated, err)
|
||||
case errors.Is(err, ErrConflict):
|
||||
return connect.NewError(connect.CodeAlreadyExists, err)
|
||||
default:
|
||||
slog.Error("unmapped rpc error", slog.Any("err", err))
|
||||
return connect.NewError(connect.CodeInternal, errors.New("internal"))
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Connect codes map 1:1 to gRPC codes. Clients see canonical error semantics.
|
||||
|
||||
---
|
||||
|
||||
## Logging interceptor
|
||||
|
||||
```go
|
||||
func loggingInterceptor() connect.UnaryInterceptorFunc {
|
||||
return func(next connect.UnaryFunc) connect.UnaryFunc {
|
||||
return func(ctx context.Context, req connect.AnyRequest) (connect.AnyResponse, error) {
|
||||
start := time.Now()
|
||||
res, err := next(ctx, req)
|
||||
attrs := []slog.Attr{
|
||||
slog.String("proc", req.Spec().Procedure),
|
||||
slog.Duration("elapsed", time.Since(start)),
|
||||
}
|
||||
if err != nil {
|
||||
attrs = append(attrs, slog.Any("err", err))
|
||||
slog.LogAttrs(ctx, slog.LevelWarn, "rpc failed", attrs...)
|
||||
} else {
|
||||
slog.LogAttrs(ctx, slog.LevelInfo, "rpc ok", attrs...)
|
||||
}
|
||||
return res, err
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
For streaming, implement the full `connect.Interceptor` (`WrapStreamingClient`, `WrapStreamingHandler`). Pattern is identical.
|
||||
|
||||
---
|
||||
|
||||
## Server streaming
|
||||
|
||||
```go
|
||||
func (s *UserServer) StreamEvents(
|
||||
ctx context.Context,
|
||||
req *connect.Request[myservicev1.StreamEventsRequest],
|
||||
stream *connect.ServerStream[myservicev1.Event],
|
||||
) error {
|
||||
events, errs := s.svc.Subscribe(ctx, req.Msg.UserId)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case e, ok := <-events:
|
||||
if !ok { return nil }
|
||||
if err := stream.Send(eventToProto(e)); err != nil {
|
||||
return err
|
||||
}
|
||||
case err := <-errs:
|
||||
return connect.NewError(connect.CodeInternal, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Same shape as SSE in `backend-stack.md`. Connect handles HTTP/2 framing.
|
||||
|
||||
---
|
||||
|
||||
## Client
|
||||
|
||||
```go
|
||||
client := myservicev1connect.NewUserServiceClient(
|
||||
http.DefaultClient,
|
||||
"https://api.example.com",
|
||||
// Use connect.WithGRPC() if the server is grpc-go and you want strict gRPC framing.
|
||||
// Default is Connect protocol — works with Connect or gRPC servers transparently.
|
||||
)
|
||||
|
||||
res, err := client.CreateUser(ctx, connect.NewRequest(&myservicev1.CreateUserRequest{
|
||||
Email: "a@b.com",
|
||||
Username: "alice",
|
||||
Age: 30,
|
||||
}))
|
||||
if err != nil {
|
||||
var connectErr *connect.Error
|
||||
if errors.As(err, &connectErr) {
|
||||
slog.Error("rpc failed",
|
||||
slog.String("code", connectErr.Code().String()),
|
||||
slog.String("msg", connectErr.Message()))
|
||||
}
|
||||
return err
|
||||
}
|
||||
slog.Info("created", slog.String("id", res.Msg.User.Id))
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## When you genuinely need raw grpc-go
|
||||
|
||||
```go
|
||||
import "google.golang.org/grpc"
|
||||
|
||||
lis, _ := net.Listen("tcp", ":8080")
|
||||
srv := grpc.NewServer(
|
||||
grpc.UnaryInterceptor(loggingUnaryInterceptor),
|
||||
)
|
||||
myservicev1.RegisterUserServiceServer(srv, &userServer{})
|
||||
_ = srv.Serve(lis)
|
||||
```
|
||||
|
||||
The codegen is from `protoc-gen-go-grpc` (different binary from `protoc-gen-connect-go`). You can codegen **both** in the same `buf.gen.yaml` and switch by importing the right package. Most teams pick one.
|
||||
|
||||
---
|
||||
|
||||
## When NOT to use RPC at all
|
||||
|
||||
If your callers are all browsers, mobile apps, third-party developers, or the long tail of "things humans curl": **stay with REST + OpenAPI**. RPC's overhead is justified for service-to-service inside a single org. Outside that boundary, JSON over HTTP wins on debuggability.
|
||||
|
||||
`oapi-codegen/oapi-codegen/v2` generates Go server stubs and clients from OpenAPI 3 — the REST equivalent of what Connect does for proto. Same parse-don't-validate boundary discipline, different wire format.
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- Connect docs: https://connectrpc.com/docs/go/getting-started
|
||||
- Buf: https://buf.build/docs
|
||||
- protovalidate: https://github.com/bufbuild/protovalidate
|
||||
- "Why we replaced protoc with buf" (Buf blog): https://buf.build/blog
|
||||
- gRPC vs Connect comparison: https://connectrpc.com/docs/introduction
|
||||
@@ -0,0 +1,337 @@
|
||||
# Library Defaults — Full Decision Tree (Go 2026)
|
||||
|
||||
The opinionated, in-production stack for 2026 Go. Every entry has a one-line rationale and a canonical snippet so the agent does not relearn each library's idioms.
|
||||
|
||||
The biggest difference from Python/Rust/TypeScript: **Go has fewer "best" choices and more "boring" choices.** The standard library is the default; reach outside it only when the rationale below applies.
|
||||
|
||||
---
|
||||
|
||||
## HTTP framework — `gin` (default) or `chi` (minimalist) or `net/http` (no deps)
|
||||
|
||||
The reality of 2026 Go: **`gin` runs ~48% of new Go API projects** (Go Developer Survey 2024 + crawls of new repos), with `gorilla/mux` (~17%, in maintenance), `echo` (~16%), and `fiber` (~11%) the remaining quarter. The skill picks gin not because it is technically superior — it is not — but because:
|
||||
|
||||
1. The ecosystem (middleware, examples, SO answers) is largest.
|
||||
2. The CLIProxyAPI codebase, which this skill's `backend-stack.md` is distilled from, uses gin in production for OpenAI/Gemini/Claude proxying including SSE streaming and WebSocket upgrades. That is real reference code, not a toy.
|
||||
3. Gin's `Context` API is the closest thing Go has to a framework-blessed "request-scoped object", which makes middleware composition straightforward.
|
||||
|
||||
```go
|
||||
import "github.com/gin-gonic/gin"
|
||||
|
||||
func main() {
|
||||
r := gin.New()
|
||||
r.Use(gin.Recovery(), middleware.RequestLogger(), middleware.RequestID())
|
||||
r.GET("/healthz", func(c *gin.Context) { c.JSON(200, gin.H{"ok": true}) })
|
||||
_ = r.Run(":8080")
|
||||
}
|
||||
```
|
||||
|
||||
**Pick `chi` instead** when:
|
||||
- You want `net/http`-compatible handlers (you do, eventually — chi is closer to stdlib).
|
||||
- The service is small and you do not need gin's binding helpers.
|
||||
|
||||
**Pick `net/http` (stdlib) directly** when:
|
||||
- The service has fewer than 10 routes and zero auth complexity. Go 1.22's enhanced `ServeMux` (method+path patterns) eliminated 80% of the historical reason to use a framework.
|
||||
|
||||
**Never use** `gorilla/mux` (effectively in maintenance), `fiber` (uses `fasthttp` which is **not stdlib-compatible**, so middleware ecosystem is split), or `echo` (smaller eco than gin, no real advantage today).
|
||||
|
||||
See `backend-stack.md` for the gin canonical layout, middleware ordering, SSE, graceful shutdown, structured logging integration.
|
||||
|
||||
---
|
||||
|
||||
## RPC — `connectrpc/connect-go`
|
||||
|
||||
The default RPC layer. **Use Connect, not raw grpc-go**, unless you have a measured reason.
|
||||
|
||||
- Connect is wire-compatible with gRPC AND speaks HTTP/1.1 + HTTP/2 + Connect protocol. One server, three clients (gRPC, gRPC-Web, Connect-Web from browsers).
|
||||
- No `grpcurl` needed for debugging — `curl -H "Content-Type: application/json" -d ...` works.
|
||||
- Streaming, interceptors, deadlines, errors are first-class.
|
||||
- Buf toolchain (`buf generate`, `buf lint`, `buf breaking`) for codegen is dramatically nicer than `protoc`.
|
||||
|
||||
```go
|
||||
// Server
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle(elizav1connect.NewElizaServiceHandler(&elizaServer{}))
|
||||
_ = http.ListenAndServe(":8080", h2c.NewHandler(mux, &http2.Server{}))
|
||||
|
||||
// Client
|
||||
client := elizav1connect.NewElizaServiceClient(
|
||||
http.DefaultClient,
|
||||
"http://localhost:8080",
|
||||
)
|
||||
res, err := client.Say(ctx, connect.NewRequest(&elizav1.SayRequest{Sentence: "hi"}))
|
||||
```
|
||||
|
||||
**Use raw `grpc-go`** only when:
|
||||
- You need server-streaming-from-multiple-services with a single gRPC mux.
|
||||
- You are integrating with a strict gRPC-only environment (Envoy proxy with gRPC reflection, Istio strict-gRPC).
|
||||
|
||||
See `grpc-connect.md`.
|
||||
|
||||
---
|
||||
|
||||
## Database — `pgx/v5` + `sqlc` + `goose`
|
||||
|
||||
```bash
|
||||
go get github.com/jackc/pgx/v5
|
||||
go install github.com/sqlc-dev/sqlc/cmd/sqlc@latest
|
||||
go install github.com/pressly/goose/v3/cmd/goose@latest
|
||||
```
|
||||
|
||||
- **`pgx/v5`** is faster, more type-safe, and has better PostgreSQL feature coverage than `database/sql + lib/pq`. Use the `pgxpool` package for connection pooling. Avoid `database/sql` driver mode — it loses pgx's batch, COPY, listen/notify.
|
||||
- **`sqlc`** generates type-safe Go from `.sql` files. Hand-written SQL with hand-written struct mapping is the #1 source of subtle DB bugs. sqlc eliminates the class.
|
||||
- **`goose`** for migrations — small, command-line first, no global state.
|
||||
|
||||
**Never use** `gorm` (active record, slow, brings runtime reflection into hot paths, encourages N+1 queries). **Never use** `ent` (heavy, opinionated graph layer) unless you specifically want a graph-shaped data model.
|
||||
|
||||
See `sqlc-pgx.md`.
|
||||
|
||||
---
|
||||
|
||||
## Validation — three layers, three tools
|
||||
|
||||
Go has no Pydantic / Zod equivalent and **does not need one** — but only because you wire three layers properly:
|
||||
|
||||
| Layer | Tool | Pattern |
|
||||
|---|---|---|
|
||||
| HTTP boundary (gin/chi/net/http) | `go-playground/validator/v10` via struct tags | `binding:"required,email,min=3"` |
|
||||
| RPC boundary (protobuf) | `bufbuild/protovalidate-go` | `(buf.validate.field).string.min_len = 3` in `.proto` |
|
||||
| Domain core | **Smart constructor + unexported fields** | `NewEmail(s) (Email, error)` returns a type whose fields cannot be set from outside |
|
||||
|
||||
```go
|
||||
// HTTP boundary
|
||||
type CreateUserReq struct {
|
||||
Email string `json:"email" binding:"required,email"`
|
||||
Username string `json:"username" binding:"required,alphanum,min=3,max=32"`
|
||||
}
|
||||
|
||||
// Domain — once a value is of type Email it is provably valid
|
||||
type Email struct{ raw string }
|
||||
func NewEmail(s string) (Email, error) {
|
||||
if !emailRegex.MatchString(s) { return Email{}, ErrInvalidEmail }
|
||||
return Email{raw: strings.ToLower(s)}, nil
|
||||
}
|
||||
func (e Email) String() string { return e.raw }
|
||||
```
|
||||
|
||||
The boundary parses raw input into the domain type **once**. Inside the domain, no further validation is permitted — the types prove it. This is parse-don't-validate adapted to Go.
|
||||
|
||||
See `data-modeling.md` for the full pattern.
|
||||
|
||||
---
|
||||
|
||||
## Logging — `log/slog` (stdlib)
|
||||
|
||||
```go
|
||||
import "log/slog"
|
||||
|
||||
logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
|
||||
Level: slog.LevelInfo,
|
||||
AddSource: true,
|
||||
}))
|
||||
slog.SetDefault(logger)
|
||||
|
||||
slog.InfoContext(ctx, "request handled",
|
||||
slog.String("path", r.URL.Path),
|
||||
slog.Int("status", 200),
|
||||
slog.Duration("elapsed", elapsed),
|
||||
)
|
||||
```
|
||||
|
||||
- **stdlib since 1.21**, stable since 1.23. Performance is on par with zerolog for structured output, and faster than logrus by a wide margin.
|
||||
- The `slog.Handler` interface is implemented by all major exporters (OpenTelemetry, Datadog, Honeycomb).
|
||||
- The skill bans `logrus`, `zap`, `zerolog` for new code. They are not bad — they are simply superseded. Existing projects on those keep them; new files use slog.
|
||||
|
||||
Use the `sloglint` linter from `golangci-strict.md` to enforce attr style (`slog.String(...)` instead of `slog.Any(...)`).
|
||||
|
||||
---
|
||||
|
||||
## CLI — `cobra` + `pflag` + slog
|
||||
|
||||
```bash
|
||||
go install github.com/spf13/cobra-cli@latest
|
||||
cobra-cli init mytool
|
||||
cobra-cli add server
|
||||
```
|
||||
|
||||
`cobra` is the de facto Go CLI framework — Kubernetes, Docker CLI, Helm, GitHub CLI all use it. The companion `viper` for config-file-+-env-+-flag merging is **optional**: prefer `caarlos0/env/v11` for env-only configs (12-factor apps), reach for viper only when you genuinely need file-based config.
|
||||
|
||||
See `cobra-stack.md`.
|
||||
|
||||
---
|
||||
|
||||
## TUI — `bubbletea v2` + `bubbles v2` + `lipgloss v2`
|
||||
|
||||
Use **v2 RC** (`charm.land/bubbletea/v2`), not v1. The v2 model adds:
|
||||
|
||||
- `tea.View{Cursor: *tea.Cursor, ...}` for real-cursor positioning.
|
||||
- `SetVirtualCursor(false)` on textareas — lets the terminal own the cursor, which is **required** for CJK IME (Korean Hangul composition, Japanese kana→kanji conversion, Chinese pinyin lookup).
|
||||
- Granular mouse events (`MouseClickMsg`, `MouseMotionMsg`, `MouseReleaseMsg`) instead of v1's coarse `MouseMsg`.
|
||||
|
||||
This is not a preference. v1 has no way to position the IME candidate window correctly — Korean input shows up two cells to the left of where you typed, every time. **If your TUI accepts text input AND your users include CJK speakers, v1 is broken.**
|
||||
|
||||
See `bubbletea-v2.md` for the full IME-correct skeleton.
|
||||
|
||||
---
|
||||
|
||||
## HTTP client — stdlib + `hashicorp/go-retryablehttp`
|
||||
|
||||
Default: `net/http.Client` with a tuned `http.Transport`. The stdlib client is **already excellent** in 2026 — HTTP/2 by default, connection pooling, sane timeouts when configured.
|
||||
|
||||
```go
|
||||
client := &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConns: 200,
|
||||
MaxIdleConnsPerHost: 40,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
DisableCompression: false,
|
||||
ForceAttemptHTTP2: true,
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
For retry/backoff, add `github.com/hashicorp/go-retryablehttp` — small, single-purpose, integrates as a wrapper.
|
||||
|
||||
**Never use** `resty` (too much magic, hides headers, encourages wrong defaults). `req` is fine but adds dependency surface for marginal benefit over the stdlib + retry wrapper.
|
||||
|
||||
---
|
||||
|
||||
## JSON — stdlib (default), `goccy/go-json` (perf), `bytedance/sonic` (extreme perf)
|
||||
|
||||
Stdlib `encoding/json` improved dramatically in Go 1.21+. **Use it.**
|
||||
|
||||
Reach for `goccy/go-json` (~3x faster) only when you have measured a hot-path bottleneck:
|
||||
|
||||
```go
|
||||
import json "github.com/goccy/go-json"
|
||||
// drop-in replacement — same API
|
||||
```
|
||||
|
||||
Reach for `bytedance/sonic` (~5x faster, requires amd64/arm64) for production proxies with thousands of RPS of JSON traversal. CLIProxyAPI uses `tidwall/gjson` + `tidwall/sjson` for **partial-tree mutation without full unmarshal** — a different optimization, useful when you transform large payloads. See `backend-stack.md`.
|
||||
|
||||
---
|
||||
|
||||
## Concurrency primitives — stdlib only
|
||||
|
||||
| Need | Use |
|
||||
|---|---|
|
||||
| Goroutine group with error propagation | `golang.org/x/sync/errgroup` |
|
||||
| Semaphore | `golang.org/x/sync/semaphore` |
|
||||
| Single-flight dedup | `golang.org/x/sync/singleflight` |
|
||||
| Lazy init | **`sync.OnceValue` / `sync.OnceFunc`** (Go 1.21+, replaces `sync.Once` for typed values) |
|
||||
| Atomic counter | `atomic.Int64` (Go 1.19+, typed atomics — don't use the old func-style) |
|
||||
| Channel-based fanout | `chan T` with `errgroup` for shutdown |
|
||||
|
||||
The `x/sync` packages are stdlib-quality but live outside `std`. See `concurrency.md` for the discipline.
|
||||
|
||||
---
|
||||
|
||||
## Time — stdlib + `benbjohnson/clock` for tests
|
||||
|
||||
```go
|
||||
type Clock interface { Now() time.Time }
|
||||
// Production
|
||||
var realClock Clock = clockImpl{}
|
||||
// Test
|
||||
fake := clock.NewMock()
|
||||
fake.Set(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||
```
|
||||
|
||||
**Never call `time.Now()` directly inside domain code.** Inject a `Clock`. Tests become deterministic, no `time.Sleep` flakiness.
|
||||
|
||||
---
|
||||
|
||||
## IDs — `google/uuid` (UUID v4/v7) or `xid` (sortable short ID)
|
||||
|
||||
```go
|
||||
import "github.com/google/uuid"
|
||||
id := uuid.Must(uuid.NewV7()) // sortable, time-ordered, 128-bit
|
||||
```
|
||||
|
||||
UUID v7 is the modern default — sortable like v6, random like v4. Use v4 only when leaking creation time is a privacy concern.
|
||||
|
||||
For short, URL-safe IDs (~12 bytes, sortable) use `rs/xid` — Kubernetes-style.
|
||||
|
||||
---
|
||||
|
||||
## Crypto — stdlib + `alecthomas/argon2id` for passwords
|
||||
|
||||
Stdlib `crypto/*` for everything. For password hashing, **argon2id is the 2026 standard** — bcrypt is acceptable but argon2 is OWASP's recommendation since 2023.
|
||||
|
||||
```go
|
||||
import "github.com/alecthomas/argon2id"
|
||||
hash, err := argon2id.CreateHash("password", argon2id.DefaultParams)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Data — `apache/arrow-go/v18` + `marcboeker/go-duckdb` + `gonum`
|
||||
|
||||
Same philosophy as Python's "never pandas":
|
||||
|
||||
| Need | Use |
|
||||
|---|---|
|
||||
| Tabular over CSV/Parquet/JSON | DuckDB-Go bindings — zero-copy Arrow integration |
|
||||
| In-memory frame | Arrow + custom code (Go has no pandas-equivalent and that's fine) |
|
||||
| Numerical | `gonum.org/v1/gonum` |
|
||||
| Stats | `gonum/stat` |
|
||||
|
||||
Go's data-science story is intentionally thin. For heavy data work, write the pipeline in Polars/DuckDB (see `python/data-processing.md`), expose the result via Parquet or Arrow, consume from Go.
|
||||
|
||||
---
|
||||
|
||||
## Testing — stdlib + selective additions
|
||||
|
||||
| Need | Use |
|
||||
|---|---|
|
||||
| Assertions | `stretchr/testify/require` (fail-fast) — `assert` only in table-driven loops |
|
||||
| Snapshots / golden | `hexops/autogold/v2` (auto-updates with `-update`) |
|
||||
| Property-based | `pgregory.net/rapid` (modern) or stdlib `testing/quick` |
|
||||
| Mocks | `go.uber.org/mock` (gomock successor) |
|
||||
| HTTP mocks | `h2non/gock` for outbound, stdlib `httptest` for inbound |
|
||||
| Integration containers | `testcontainers/testcontainers-go` |
|
||||
| Goroutine leak | `go.uber.org/goleak` |
|
||||
| Benchmarks | stdlib `testing.B` + `perf.dev/benchstat` |
|
||||
|
||||
See `testing.md` for canonical patterns.
|
||||
|
||||
---
|
||||
|
||||
## Config — `caarlos0/env/v11`
|
||||
|
||||
```go
|
||||
type Config struct {
|
||||
Port int `env:"PORT" envDefault:"8080"`
|
||||
DatabaseURL string `env:"DATABASE_URL,required"`
|
||||
Timeout time.Duration `env:"TIMEOUT" envDefault:"30s"`
|
||||
}
|
||||
|
||||
var cfg Config
|
||||
if err := env.Parse(&cfg); err != nil { log.Fatal(err) }
|
||||
```
|
||||
|
||||
Pure 12-factor. Defaults via struct tag, required marker, parsing for `time.Duration`, slices, maps. **Use viper only if you also need file-based config** — most services do not.
|
||||
|
||||
---
|
||||
|
||||
## Choosing an unfamiliar dependency — the checklist
|
||||
|
||||
Before `go get`-ing anything new:
|
||||
|
||||
1. Is it maintained? Latest tag within 12 months? Owner active?
|
||||
2. Does it expose stdlib-compatible types (`io.Reader`, `context.Context`, `http.Handler`)? If it invents its own `Connection` or `Request` type, that's a yellow flag.
|
||||
3. Does it use `init()` for side effects? **REJECT.** `init()` ruins testability.
|
||||
4. Does it call `log.Fatal` / `panic` outside of true programmer-error paths? **REJECT.**
|
||||
5. Does it have a `context.Context` first-arg convention? If not, **REJECT** — cancellation is non-negotiable.
|
||||
6. Does adding it overlap with something already in your `go.mod`? Pick one.
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- 2024 Go Developer Survey: https://go.dev/blog/survey2024-h1-results
|
||||
- Connect-Go docs: https://connectrpc.com/docs/go/getting-started
|
||||
- sqlc: https://docs.sqlc.dev
|
||||
- bubbletea v2 IME: https://github.com/code-yeongyu/bubbletea-wm (reference for `SetVirtualCursor(false)` pattern)
|
||||
- CLIProxyAPI (gin + SSE + WebSocket in production): https://github.com/router-for-me/CLIProxyAPI
|
||||
- slog blog: https://go.dev/blog/slog
|
||||
@@ -0,0 +1,202 @@
|
||||
# One-Liners and Disposable Scripts
|
||||
|
||||
Production hygiene with throwaway ergonomics. Go scripts get the same strict lints, the same type discipline, the same 250 LOC ceiling. The difference: they live as single `.go` files invoked via `go run`, not as full modules.
|
||||
|
||||
Python has PEP 723 + `uv run`. Rust has `rust-script`. **Go has `go run` directly** — no extra tooling needed.
|
||||
|
||||
---
|
||||
|
||||
## Pattern 1: Single-file `go run`
|
||||
|
||||
A `.go` file with a `main` package, run directly:
|
||||
|
||||
```go
|
||||
//go:build ignore
|
||||
// fetch.go — fetch a URL and print body length.
|
||||
//
|
||||
// Usage:
|
||||
// go run fetch.go <url>
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
log.Fatal("usage: go run fetch.go <url>")
|
||||
}
|
||||
resp, err := http.Get(os.Args[1])
|
||||
if err != nil { log.Fatal(err) }
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil { log.Fatal(err) }
|
||||
|
||||
fmt.Printf("%d bytes\n", len(body))
|
||||
}
|
||||
```
|
||||
|
||||
Run: `go run fetch.go https://example.com`.
|
||||
|
||||
The `//go:build ignore` directive keeps this file out of `go build ./...` — it is a script, not part of the module. Without that line, every `.go` file in the package gets compiled into your binary.
|
||||
|
||||
---
|
||||
|
||||
## Pattern 2: Throwaway directory under `scripts/`
|
||||
|
||||
```
|
||||
myproject/
|
||||
├── go.mod
|
||||
├── internal/...
|
||||
└── scripts/
|
||||
├── seed/
|
||||
│ └── main.go # `go run ./scripts/seed`
|
||||
├── migrate/
|
||||
│ └── main.go
|
||||
└── one-time-fix/
|
||||
└── main.go
|
||||
```
|
||||
|
||||
Each `scripts/<name>/main.go` is its own `main` package. Invoke as `go run ./scripts/seed/`. Dependencies are shared with the parent module — no separate `go.mod`.
|
||||
|
||||
This is the right pattern when:
|
||||
|
||||
- You need module deps (sqlc, pgx, your own internal packages).
|
||||
- You want IDE support, type-checking, test coverage.
|
||||
- The script lives alongside the project, runs in CI.
|
||||
|
||||
---
|
||||
|
||||
## Pattern 3: Inline `go run` from shell
|
||||
|
||||
```bash
|
||||
go run -mod=mod <(cat <<'EOF'
|
||||
package main
|
||||
import "fmt"
|
||||
func main() { fmt.Println("hello") }
|
||||
EOF
|
||||
)
|
||||
```
|
||||
|
||||
Rare, but useful for one-shot terminal experiments. The `<(...)` is process substitution; `go run -mod=mod` reads from stdin.
|
||||
|
||||
---
|
||||
|
||||
## Hard rules for scripts
|
||||
|
||||
Even a 30-line script follows the philosophy:
|
||||
|
||||
1. **Typed flags via `flag` or `pflag`**, not `os.Args` string parsing past 2 args.
|
||||
```go
|
||||
var (
|
||||
url = flag.String("url", "", "URL to fetch")
|
||||
limit = flag.Int("limit", 100, "max bytes")
|
||||
)
|
||||
flag.Parse()
|
||||
if *url == "" { log.Fatal("--url required") }
|
||||
```
|
||||
|
||||
2. **`context.Context` propagation** wherever I/O happens.
|
||||
```go
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
req, _ := http.NewRequestWithContext(ctx, "GET", *url, nil)
|
||||
```
|
||||
|
||||
3. **`log.Fatal` is fine in `main()`** of a script (programmer error / fatal path), but **never inside any function the script imports.** Library code returns errors.
|
||||
|
||||
4. **Errors get wrapped.** Same rule as production code:
|
||||
```go
|
||||
if err != nil { return fmt.Errorf("fetch %s: %w", *url, err) }
|
||||
```
|
||||
|
||||
5. **Resources released via `defer`.** No "I'll fix it later".
|
||||
|
||||
6. **slog for output if it must be parseable.** `fmt.Println` for one-shot terminal output is fine.
|
||||
|
||||
7. **No more than 250 pure LOC.** If it grows, it stops being a script and becomes a subcommand of your CLI tool.
|
||||
|
||||
---
|
||||
|
||||
## Pattern 4: Standalone tool with deps — temporary module
|
||||
|
||||
Some scripts need deps the parent module does not have. Two options:
|
||||
|
||||
### Option A — script in its own tiny module
|
||||
|
||||
```bash
|
||||
mkdir /tmp/migrate-tool && cd $_
|
||||
go mod init scratch.local/migrate-tool
|
||||
go get github.com/pressly/goose/v3
|
||||
cat > main.go <<'EOF'
|
||||
package main
|
||||
import ... // use goose
|
||||
func main() { ... }
|
||||
EOF
|
||||
go run .
|
||||
```
|
||||
|
||||
Run, then delete `/tmp/migrate-tool`. Throwaway.
|
||||
|
||||
### Option B — `gorun` (community tool)
|
||||
|
||||
```bash
|
||||
go install github.com/erning/gorun@latest
|
||||
|
||||
cat > script.go <<'EOF'
|
||||
//usr/bin/env gorun "$0" "$@"; exit
|
||||
// /// go.mod
|
||||
// module scratch
|
||||
// go 1.23
|
||||
// require github.com/spf13/cobra v1.8.0
|
||||
// ///
|
||||
|
||||
package main
|
||||
...
|
||||
EOF
|
||||
chmod +x script.go
|
||||
./script.go
|
||||
```
|
||||
|
||||
`gorun` parses the inline `go.mod` block, materializes a temp module, runs the script. Niche tool — only if you want the executable-script experience.
|
||||
|
||||
---
|
||||
|
||||
## When a script becomes a CLI
|
||||
|
||||
If your script needs:
|
||||
|
||||
- More than one subcommand
|
||||
- Long-term storage of state
|
||||
- Help text more than a paragraph
|
||||
- Repeated invocations from CI
|
||||
|
||||
... promote it to a real CLI tool via `cobra` — see `cobra-stack.md`. The boundary is fuzzy; trust your judgment, but **a 500-line "script" is not a script.**
|
||||
|
||||
---
|
||||
|
||||
## Antipatterns
|
||||
|
||||
| Bad | Why | Good |
|
||||
|---|---|---|
|
||||
| `os.Args[1]` indexing without length check | Panics on missing arg | `flag.Parse()` with explicit checks |
|
||||
| `log.Fatal` inside a function the script imports | Crashes caller's process | Return error |
|
||||
| `panic(err)` for expected failures | Same as above | `log.Fatal` in `main`, error return elsewhere |
|
||||
| Skipping `defer resp.Body.Close()` because "it's a script" | Leaks fd | Always close |
|
||||
| One 800-LOC `main.go` "to keep it simple" | Now harder to read than a real CLI | Promote to `cmd/<name>/` with subcommands |
|
||||
| `// TODO: handle error` | Production-grade hygiene means production-grade hygiene | Handle now or document why ignored |
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- `go run` docs: https://pkg.go.dev/cmd/go#hdr-Compile_and_run_Go_program
|
||||
- `//go:build` constraints: https://pkg.go.dev/cmd/go#hdr-Build_constraints
|
||||
- `signal.NotifyContext`: https://pkg.go.dev/os/signal#NotifyContext
|
||||
- gorun: https://github.com/erning/gorun
|
||||
@@ -0,0 +1,471 @@
|
||||
# Database Stack — sqlc + pgx + goose + testcontainers
|
||||
|
||||
The canonical 2026 PostgreSQL stack. **Type-safe SQL with zero runtime reflection**, hot-path-friendly connection pooling, sane migrations, real Postgres in tests.
|
||||
|
||||
If you came here from a `gorm` project: gorm is rejected. See "Why not gorm" at the end.
|
||||
|
||||
---
|
||||
|
||||
## Toolchain
|
||||
|
||||
```bash
|
||||
go install github.com/sqlc-dev/sqlc/cmd/sqlc@latest
|
||||
go install github.com/pressly/goose/v3/cmd/goose@latest
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
internal/store/
|
||||
├── sqlc.yaml # sqlc config
|
||||
├── schema.sql # the cumulative DDL sqlc parses
|
||||
├── queries/ # one *.sql per resource
|
||||
│ ├── users.sql
|
||||
│ ├── orders.sql
|
||||
│ └── sessions.sql
|
||||
├── sqlc/ # GENERATED — do not hand-edit
|
||||
│ ├── db.go
|
||||
│ ├── models.go
|
||||
│ ├── users.sql.go
|
||||
│ ├── orders.sql.go
|
||||
│ └── sessions.sql.go
|
||||
├── migrations/ # goose migrations, ordered
|
||||
│ ├── 20260101000001_create_users.sql
|
||||
│ └── 20260102000001_add_orders.sql
|
||||
├── pool.go # pgxpool factory
|
||||
├── user_store.go # domain-facing wrapper around sqlc
|
||||
└── user_store_test.go # testcontainers integration test
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## `sqlc.yaml`
|
||||
|
||||
```yaml
|
||||
version: "2"
|
||||
sql:
|
||||
- engine: "postgresql"
|
||||
schema: "schema.sql"
|
||||
queries: "queries"
|
||||
gen:
|
||||
go:
|
||||
package: "sqlc"
|
||||
out: "sqlc"
|
||||
sql_package: "pgx/v5"
|
||||
emit_json_tags: false
|
||||
emit_prepared_queries: false
|
||||
emit_interface: true # generates a Querier interface
|
||||
emit_exact_table_names: false
|
||||
emit_pointers_for_null_types: true
|
||||
emit_empty_slices: true
|
||||
overrides:
|
||||
- db_type: "uuid"
|
||||
go_type:
|
||||
import: "github.com/google/uuid"
|
||||
type: "UUID"
|
||||
- db_type: "timestamptz"
|
||||
go_type:
|
||||
import: "time"
|
||||
type: "Time"
|
||||
```
|
||||
|
||||
Key choices:
|
||||
|
||||
- `sql_package: "pgx/v5"` — generated code uses pgx directly, not `database/sql`. Faster, type-safer.
|
||||
- `emit_interface: true` — generates a `Querier` interface. Lets stores accept either `*pgxpool.Pool` or `pgx.Tx` for transaction support.
|
||||
- `emit_pointers_for_null_types: true` — nullable columns become `*T`, not `sql.NullString`. Cleaner mapping to domain types.
|
||||
- `overrides` for `uuid` → `google/uuid.UUID` and `timestamptz` → `time.Time`.
|
||||
|
||||
---
|
||||
|
||||
## `schema.sql`
|
||||
|
||||
```sql
|
||||
-- internal/store/schema.sql
|
||||
-- The CUMULATIVE schema sqlc parses. Not migrations — the end state.
|
||||
-- Regenerate from a fresh DB via `pg_dump --schema-only`, or hand-maintain.
|
||||
|
||||
CREATE TABLE users (
|
||||
id UUID PRIMARY KEY,
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX idx_users_created_at ON users(created_at DESC);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## `queries/users.sql`
|
||||
|
||||
```sql
|
||||
-- name: GetUser :one
|
||||
SELECT id, email, username, created_at
|
||||
FROM users
|
||||
WHERE id = $1;
|
||||
|
||||
-- name: ListUsers :many
|
||||
SELECT id, email, username, created_at
|
||||
FROM users
|
||||
ORDER BY created_at DESC
|
||||
LIMIT $1 OFFSET $2;
|
||||
|
||||
-- name: CreateUser :one
|
||||
INSERT INTO users (id, email, username)
|
||||
VALUES ($1, $2, $3)
|
||||
RETURNING id, email, username, created_at;
|
||||
|
||||
-- name: UpdateUserEmail :exec
|
||||
UPDATE users
|
||||
SET email = $2
|
||||
WHERE id = $1;
|
||||
|
||||
-- name: DeleteUser :exec
|
||||
DELETE FROM users WHERE id = $1;
|
||||
```
|
||||
|
||||
sqlc directives:
|
||||
|
||||
- `:one` — exactly one row; returns `(T, error)`. Returns `pgx.ErrNoRows` on miss.
|
||||
- `:many` — zero or more rows; returns `([]T, error)`.
|
||||
- `:exec` — no rows returned; returns `error`.
|
||||
- `:execrows` — returns `(int64, error)` with affected row count.
|
||||
- `:batchone` / `:batchmany` / `:batchexec` — pgx batch mode for bulk operations.
|
||||
|
||||
Run `task gen:sqlc` (or `sqlc generate`). The generated file is committed; CI checks it is up-to-date.
|
||||
|
||||
---
|
||||
|
||||
## Generated code shape (`sqlc/users.sql.go`)
|
||||
|
||||
```go
|
||||
// GENERATED — do not edit
|
||||
type User struct {
|
||||
ID uuid.UUID
|
||||
Email string
|
||||
Username string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
const getUser = `-- name: GetUser :one
|
||||
SELECT id, email, username, created_at FROM users WHERE id = $1`
|
||||
|
||||
func (q *Queries) GetUser(ctx context.Context, id uuid.UUID) (User, error) {
|
||||
row := q.db.QueryRow(ctx, getUser, id)
|
||||
var u User
|
||||
err := row.Scan(&u.ID, &u.Email, &u.Username, &u.CreatedAt)
|
||||
return u, err
|
||||
}
|
||||
```
|
||||
|
||||
Type-safe inputs, type-safe outputs, compile-time-checked column-to-field mapping. **A schema change that drops a column breaks compilation.** Hand-rolled SQL would have failed at runtime.
|
||||
|
||||
---
|
||||
|
||||
## `store/pool.go`
|
||||
|
||||
```go
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func NewPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) {
|
||||
cfg, err := pgxpool.ParseConfig(dsn)
|
||||
if err != nil { return nil, fmt.Errorf("parse dsn: %w", err) }
|
||||
|
||||
cfg.MaxConns = 25
|
||||
cfg.MinConns = 5
|
||||
cfg.MaxConnLifetime = time.Hour
|
||||
cfg.MaxConnIdleTime = 30 * time.Minute
|
||||
cfg.HealthCheckPeriod = 1 * time.Minute
|
||||
|
||||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil { return nil, fmt.Errorf("connect: %w", err) }
|
||||
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("ping: %w", err)
|
||||
}
|
||||
return pool, nil
|
||||
}
|
||||
```
|
||||
|
||||
`pgxpool.Pool` is `Querier`-compatible (implements the interface sqlc generates). Same pool flows into sqlc queries unchanged.
|
||||
|
||||
---
|
||||
|
||||
## `store/user_store.go` — domain ↔ sqlc
|
||||
|
||||
```go
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/your-org/myservice/internal/domain"
|
||||
"github.com/your-org/myservice/internal/store/sqlc"
|
||||
)
|
||||
|
||||
type UserStore struct {
|
||||
q *sqlc.Queries
|
||||
}
|
||||
|
||||
func NewUserStore(pool *pgxpool.Pool) *UserStore {
|
||||
return &UserStore{q: sqlc.New(pool)}
|
||||
}
|
||||
|
||||
func (s *UserStore) Get(ctx context.Context, id domain.UserID) (domain.User, error) {
|
||||
row, err := s.q.GetUser(ctx, uuid.UUID(id))
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return domain.User{}, domain.ErrUserNotFound
|
||||
}
|
||||
return domain.User{}, fmt.Errorf("get user %s: %w", id, err)
|
||||
}
|
||||
return rowToDomain(row)
|
||||
}
|
||||
|
||||
func (s *UserStore) Create(ctx context.Context, u domain.User) (domain.User, error) {
|
||||
row, err := s.q.CreateUser(ctx, sqlc.CreateUserParams{
|
||||
ID: uuid.UUID(u.ID),
|
||||
Email: u.Email.String(),
|
||||
Username: u.Username.String(),
|
||||
})
|
||||
if err != nil {
|
||||
return domain.User{}, fmt.Errorf("create user: %w", err)
|
||||
}
|
||||
return rowToDomain(row)
|
||||
}
|
||||
|
||||
func rowToDomain(r sqlc.User) (domain.User, error) {
|
||||
email, err := domain.NewEmail(r.Email)
|
||||
if err != nil {
|
||||
return domain.User{}, fmt.Errorf("db invariant: email %q: %w", r.Email, err)
|
||||
}
|
||||
username, err := domain.NewUsername(r.Username)
|
||||
if err != nil {
|
||||
return domain.User{}, fmt.Errorf("db invariant: username %q: %w", r.Username, err)
|
||||
}
|
||||
return domain.User{
|
||||
ID: domain.UserID(r.ID),
|
||||
Email: email,
|
||||
Username: username,
|
||||
CreatedAt: r.CreatedAt,
|
||||
}, nil
|
||||
}
|
||||
```
|
||||
|
||||
The wrapping is verbose. **That is the point.** sqlc rows are storage representations; domain types are business representations. Mapping them explicitly is where invariants are enforced.
|
||||
|
||||
`pgx.ErrNoRows` becomes `domain.ErrUserNotFound` — callers never see storage-level errors.
|
||||
|
||||
---
|
||||
|
||||
## Transactions — pgx.Tx satisfies the Querier interface
|
||||
|
||||
```go
|
||||
func (s *UserStore) CreateWithProfile(
|
||||
ctx context.Context,
|
||||
pool *pgxpool.Pool,
|
||||
u domain.User,
|
||||
p domain.Profile,
|
||||
) error {
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil { return fmt.Errorf("begin: %w", err) }
|
||||
defer tx.Rollback(ctx) // no-op if Commit succeeded
|
||||
|
||||
q := s.q.WithTx(tx) // sqlc.Queries bound to the tx
|
||||
|
||||
if _, err := q.CreateUser(ctx, /* ... */); err != nil {
|
||||
return fmt.Errorf("create user: %w", err)
|
||||
}
|
||||
if _, err := q.CreateProfile(ctx, /* ... */); err != nil {
|
||||
return fmt.Errorf("create profile: %w", err)
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
```
|
||||
|
||||
Pattern:
|
||||
|
||||
- `defer tx.Rollback(ctx)` immediately after `Begin` — safe even after Commit (returns "tx closed", which we ignore via the unhandled return).
|
||||
- `q.WithTx(tx)` returns a `*Queries` bound to the tx.
|
||||
- Last line: `tx.Commit(ctx)`.
|
||||
|
||||
For nested transactions across multiple stores, accept a `Querier` parameter:
|
||||
|
||||
```go
|
||||
func (s *UserStore) CreateTx(ctx context.Context, q sqlc.Querier, u domain.User) (domain.User, error) {
|
||||
// uses q instead of s.q — caller decides if it's pool or tx
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Migrations — goose
|
||||
|
||||
```bash
|
||||
goose -dir internal/store/migrations create create_users sql
|
||||
```
|
||||
|
||||
```sql
|
||||
-- migrations/20260101000001_create_users.sql
|
||||
-- +goose Up
|
||||
CREATE TABLE users (
|
||||
id UUID PRIMARY KEY,
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE users;
|
||||
```
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
goose -dir internal/store/migrations postgres "$DATABASE_URL" up
|
||||
goose -dir internal/store/migrations postgres "$DATABASE_URL" status
|
||||
goose -dir internal/store/migrations postgres "$DATABASE_URL" down
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- One DDL change per migration. Never combine schema + data migrations in one file.
|
||||
- `Down` is real, not a stub. CI runs `up` → `down` → `up` on a fresh container to prove reversibility.
|
||||
- Migrations are append-only. Never edit a merged migration; add a new one.
|
||||
|
||||
`goose` can run programmatically as well:
|
||||
|
||||
```go
|
||||
import "github.com/pressly/goose/v3"
|
||||
|
||||
if err := goose.UpContext(ctx, db, "migrations"); err != nil { ... }
|
||||
```
|
||||
|
||||
Useful for tools that own their schema (CI runner, integration test setup).
|
||||
|
||||
---
|
||||
|
||||
## Integration tests — testcontainers
|
||||
|
||||
```go
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/testcontainers/testcontainers-go/modules/postgres"
|
||||
)
|
||||
|
||||
func newTestDB(t *testing.T) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
|
||||
pgC, err := postgres.Run(ctx,
|
||||
"postgres:16-alpine",
|
||||
postgres.WithDatabase("test"),
|
||||
postgres.WithUsername("test"),
|
||||
postgres.WithPassword("test"),
|
||||
postgres.BasicWaitStrategies(),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = pgC.Terminate(ctx) })
|
||||
|
||||
dsn, err := pgC.ConnectionString(ctx, "sslmode=disable")
|
||||
require.NoError(t, err)
|
||||
|
||||
pool, err := store.NewPool(ctx, dsn)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(pool.Close)
|
||||
|
||||
require.NoError(t, goose.UpContext(ctx, /* sql.DB from pool */, "../migrations"))
|
||||
return pool
|
||||
}
|
||||
|
||||
func TestUserStore_Create_returns_new_user(t *testing.T) {
|
||||
// Given
|
||||
pool := newTestDB(t)
|
||||
s := store.NewUserStore(pool)
|
||||
ctx := context.Background()
|
||||
|
||||
// When
|
||||
user, err := s.Create(ctx, domain.User{
|
||||
ID: domain.UserID(uuid.Must(uuid.NewV7())),
|
||||
Email: mustEmail("a@b.com"),
|
||||
Username: mustUsername("alice"),
|
||||
})
|
||||
|
||||
// Then
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, user.ID)
|
||||
|
||||
fetched, err := s.Get(ctx, user.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, user.Email, fetched.Email)
|
||||
}
|
||||
```
|
||||
|
||||
testcontainers spins a real Postgres in Docker, runs migrations, hands you a pool. Tests are slow (~2s startup) but **real** — no fake that diverges from production.
|
||||
|
||||
For test suites with many cases, share one container across tests in the same package via `TestMain`:
|
||||
|
||||
```go
|
||||
var testPool *pgxpool.Pool
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
pgC, _ := postgres.Run(ctx, "postgres:16-alpine", /* ... */)
|
||||
defer pgC.Terminate(ctx)
|
||||
dsn, _ := pgC.ConnectionString(ctx, "sslmode=disable")
|
||||
testPool, _ = store.NewPool(ctx, dsn)
|
||||
// run migrations once
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
```
|
||||
|
||||
Each test then uses a transaction it rolls back at the end — fast and isolated.
|
||||
|
||||
---
|
||||
|
||||
## Why NOT gorm
|
||||
|
||||
| Concern | gorm | sqlc + pgx |
|
||||
|---|---|---|
|
||||
| Type safety | runtime reflection; column-to-field via tags | compile-time-checked from SQL |
|
||||
| Performance | 2–5x slower than pgx | pgx is the fastest Go pg driver |
|
||||
| N+1 queries | encouraged by `Preload` API | explicit JOIN in `.sql` |
|
||||
| Migrations | AutoMigrate (unsafe in prod) | goose, explicit |
|
||||
| Debugging | "what query did it run?" requires logging | the query IS the source |
|
||||
| Cancellation | spotty ctx support | first-class |
|
||||
| Active development | Yes but with churn and breaking changes | sqlc is stable |
|
||||
|
||||
Existing gorm projects: leave them. New code: sqlc + pgx.
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- sqlc docs: https://docs.sqlc.dev
|
||||
- pgx: https://github.com/jackc/pgx
|
||||
- goose: https://github.com/pressly/goose
|
||||
- testcontainers-go: https://golang.testcontainers.org
|
||||
- pgx pool config: https://pkg.go.dev/github.com/jackc/pgx/v5/pgxpool#Config
|
||||
@@ -0,0 +1,467 @@
|
||||
# Testing
|
||||
|
||||
TDD shape, table-driven tests, `require` vs `assert`, snapshot tests, property-based tests, integration tests with testcontainers, goroutine-leak detection. The discipline in `programming/SKILL.md` (Given/When/Then, less mock the better, efficient AND accurate) — this document gives the Go-specific recipes.
|
||||
|
||||
---
|
||||
|
||||
## Tools
|
||||
|
||||
| Need | Use |
|
||||
|---|---|
|
||||
| Assertions | `stretchr/testify/require` (and `assert` only inside table loops) |
|
||||
| Mocks | `go.uber.org/mock` (gomock successor) |
|
||||
| Goroutine leaks | `go.uber.org/goleak` |
|
||||
| Snapshots / golden | `hexops/autogold/v2` |
|
||||
| Property-based | `pgregory.net/rapid` |
|
||||
| HTTP mocks (outbound) | `h2non/gock` |
|
||||
| HTTP test server (inbound) | stdlib `net/http/httptest` |
|
||||
| Integration containers | `testcontainers/testcontainers-go` |
|
||||
| TUI | `charm.land/bubbletea/v2/teatest` |
|
||||
| Bench tooling | stdlib `testing.B` + `perf.dev/benchstat` |
|
||||
|
||||
---
|
||||
|
||||
## Test naming — Given / When / Then in the name
|
||||
|
||||
```go
|
||||
// ──── PATTERN ────
|
||||
// Test_<Subject>_<Outcome>_when_<Condition>
|
||||
// OR
|
||||
// Test_<Subject>_<Action>_<ExpectedOutcome>
|
||||
|
||||
func Test_Email_NewEmail_lowercases_input(t *testing.T)
|
||||
func Test_Email_NewEmail_rejects_input_without_at_sign(t *testing.T)
|
||||
func Test_UserService_Create_persists_user_when_inputs_valid(t *testing.T)
|
||||
func Test_UserService_Create_returns_validation_error_when_email_invalid(t *testing.T)
|
||||
```
|
||||
|
||||
A test name should answer "what behavior is this asserting?" without reading the body. Names that need a comment to explain them are misnamed.
|
||||
|
||||
---
|
||||
|
||||
## Single test — explicit Given/When/Then
|
||||
|
||||
```go
|
||||
func Test_Email_NewEmail_rejects_input_without_at_sign(t *testing.T) {
|
||||
// Given
|
||||
raw := "not-an-email"
|
||||
|
||||
// When
|
||||
_, err := domain.NewEmail(raw)
|
||||
|
||||
// Then
|
||||
require.Error(t, err)
|
||||
require.ErrorIs(t, err, domain.ErrInvalidEmail)
|
||||
}
|
||||
```
|
||||
|
||||
`require.*` fails the test immediately on miss. Use `require` for preconditions and primary assertions. Use `assert.*` only inside table-driven loops where you want all cases to report.
|
||||
|
||||
---
|
||||
|
||||
## Table-driven tests
|
||||
|
||||
```go
|
||||
func Test_Email_NewEmail(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
wantErr error
|
||||
}{
|
||||
{"lowercases", "ALICE@example.com", "alice@example.com", nil},
|
||||
{"trims whitespace", " bob@example.com ", "bob@example.com", nil},
|
||||
{"rejects missing @", "no-at-sign", "", domain.ErrInvalidEmail},
|
||||
{"rejects empty", "", "", domain.ErrInvalidEmail},
|
||||
{"rejects too long", strings.Repeat("a", 256) + "@e.com", "", domain.ErrInvalidEmail},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// When
|
||||
got, err := domain.NewEmail(tt.input)
|
||||
|
||||
// Then
|
||||
if tt.wantErr != nil {
|
||||
require.ErrorIs(t, err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, got.String())
|
||||
})
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- One **scenario** per row, not one **assertion** per row.
|
||||
- Subtest names are sentences in lowercase; `t.Run(tt.name, ...)` makes them filterable: `go test -run Test_Email_NewEmail/rejects_missing_@`.
|
||||
- The loop body itself is Given/When/Then in shape.
|
||||
- For Go 1.22+, the loop var capture works correctly without the `tt := tt` shadow line — the `copyloopvar` linter enforces the new style.
|
||||
|
||||
---
|
||||
|
||||
## Less mocks — the priority order
|
||||
|
||||
In Go specifically:
|
||||
|
||||
1. **Real implementation.** Domain types, pure functions, value objects — instantiate them. They are fast.
|
||||
2. **In-memory fake** that satisfies the interface. Has its own test suite proving behavioral parity with the real impl.
|
||||
3. **`httptest.Server`** for HTTP collaborators (real wire, no internet).
|
||||
4. **`testcontainers`** for stateful collaborators (Postgres, Redis, S3-compatible, Kafka).
|
||||
5. **gomock** ONLY for: clocks, randomness, third-party SaaS with no sandbox.
|
||||
|
||||
### Example: an in-memory fake
|
||||
|
||||
```go
|
||||
// Real interface
|
||||
type UserRepo interface {
|
||||
Save(ctx context.Context, u domain.User) error
|
||||
Get(ctx context.Context, id domain.UserID) (domain.User, error)
|
||||
}
|
||||
|
||||
// In-memory fake — production-quality, tested separately
|
||||
type FakeUserRepo struct {
|
||||
mu sync.RWMutex
|
||||
users map[domain.UserID]domain.User
|
||||
}
|
||||
|
||||
func NewFakeUserRepo() *FakeUserRepo {
|
||||
return &FakeUserRepo{users: map[domain.UserID]domain.User{}}
|
||||
}
|
||||
|
||||
func (r *FakeUserRepo) Save(ctx context.Context, u domain.User) error {
|
||||
r.mu.Lock(); defer r.mu.Unlock()
|
||||
r.users[u.ID] = u
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *FakeUserRepo) Get(ctx context.Context, id domain.UserID) (domain.User, error) {
|
||||
r.mu.RLock(); defer r.mu.RUnlock()
|
||||
u, ok := r.users[id]
|
||||
if !ok { return domain.User{}, domain.ErrUserNotFound }
|
||||
return u, nil
|
||||
}
|
||||
```
|
||||
|
||||
The fake has the same observable behavior as the real one. Tests against `FakeUserRepo` survive when the production repo's internals change. Tests against a gomock stub of `UserRepo` break.
|
||||
|
||||
**A test passing against a fake AND a test passing against the real impl is the gold standard.** Run the same test suite twice — once with the fake, once with testcontainers. The fakes earn their keep when the suites diverge.
|
||||
|
||||
### Example: gomock for the unmockable
|
||||
|
||||
```go
|
||||
//go:generate mockgen -source=clock.go -destination=mocks/clock_mock.go -package=mocks
|
||||
|
||||
type Clock interface {
|
||||
Now() time.Time
|
||||
}
|
||||
|
||||
// In a test:
|
||||
ctrl := gomock.NewController(t)
|
||||
clock := mocks.NewMockClock(ctrl)
|
||||
clock.EXPECT().Now().Return(fixedTime).AnyTimes()
|
||||
```
|
||||
|
||||
Mock the narrowest seam. Never mock `UserRepo` if a fake suffices.
|
||||
|
||||
---
|
||||
|
||||
## E2E scenario tests
|
||||
|
||||
```go
|
||||
//go:build e2e
|
||||
|
||||
func Test_E2E_user_can_signup_then_login(t *testing.T) {
|
||||
// Given — full server in a goroutine
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
pool := newTestDB(t) // testcontainers Postgres
|
||||
server := startServer(t, pool) // real gin engine on a random port
|
||||
defer server.Close()
|
||||
|
||||
client := server.Client()
|
||||
|
||||
// When — sign up
|
||||
resp, err := client.Post(server.URL+"/api/v1/users",
|
||||
"application/json",
|
||||
strings.NewReader(`{"email":"a@b.com","username":"alice","password":"PassWord!23"}`),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 201, resp.StatusCode)
|
||||
|
||||
// When — log in
|
||||
resp, err = client.Post(server.URL+"/api/v1/auth/login",
|
||||
"application/json",
|
||||
strings.NewReader(`{"email":"a@b.com","password":"PassWord!23"}`),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, resp.StatusCode)
|
||||
|
||||
var body struct{ Token string `json:"token"` }
|
||||
require.NoError(t, json.NewDecoder(resp.Body).Decode(&body))
|
||||
require.NotEmpty(t, body.Token)
|
||||
|
||||
// Then — token works on protected endpoint
|
||||
req, _ := http.NewRequestWithContext(ctx, "GET", server.URL+"/api/v1/me", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+body.Token)
|
||||
resp, err = client.Do(req)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, resp.StatusCode)
|
||||
}
|
||||
```
|
||||
|
||||
Patterns:
|
||||
|
||||
- `//go:build e2e` build tag separates slow E2E from fast unit tests. Run with `go test -tags=e2e ./...`.
|
||||
- One narrative per test: "user can sign up then log in". One `Test_E2E_*` per user-visible outcome.
|
||||
- Real DB via testcontainers, real gin engine, real HTTP. **No mocks.** The point is to catch integration bugs.
|
||||
- Bounded context — every E2E gets a `context.WithTimeout` so failures don't hang CI.
|
||||
|
||||
---
|
||||
|
||||
## Goroutine leak detection
|
||||
|
||||
```go
|
||||
package mypkg
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"go.uber.org/goleak"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
goleak.VerifyTestMain(m,
|
||||
goleak.IgnoreTopFunction("github.com/prometheus/client_golang/prometheus.(*Registry)..."),
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
One line at the top of every package that spawns goroutines. Catches the bug class the race detector cannot.
|
||||
|
||||
---
|
||||
|
||||
## Snapshot / golden tests — `autogold`
|
||||
|
||||
```go
|
||||
import "github.com/hexops/autogold/v2"
|
||||
|
||||
func Test_RenderHelp_matches_snapshot(t *testing.T) {
|
||||
// Given
|
||||
cmd := newRootCmd()
|
||||
|
||||
// When
|
||||
out := captureOutput(t, func() { _ = cmd.Help() })
|
||||
|
||||
// Then
|
||||
autogold.ExpectFile(t, out)
|
||||
}
|
||||
```
|
||||
|
||||
First run: `go test -update ./...` writes `testdata/Test_RenderHelp.golden`. Future runs compare; failures show a diff. Re-approve intentional changes with `-update`.
|
||||
|
||||
**Use snapshots for STRUCTURE, not BEHAVIOR.** Good targets:
|
||||
|
||||
- CLI `--help` output
|
||||
- JSON response shape
|
||||
- Generated SQL queries
|
||||
- Rendered prompts (assert the structure, not exact wording — see SKILL.md prompt-test rule)
|
||||
|
||||
Bad targets: a function's return value where you should `require.Equal` on the actual structure.
|
||||
|
||||
---
|
||||
|
||||
## Property-based tests — `rapid`
|
||||
|
||||
```go
|
||||
import "pgregory.net/rapid"
|
||||
|
||||
func Test_Email_NewEmail_then_String_roundtrips(t *testing.T) {
|
||||
rapid.Check(t, func(t *rapid.T) {
|
||||
// Given — generate valid emails
|
||||
local := rapid.StringMatching(`[a-z]{3,10}`).Draw(t, "local")
|
||||
domain := rapid.StringMatching(`[a-z]{3,10}\.com`).Draw(t, "domain")
|
||||
raw := local + "@" + domain
|
||||
|
||||
// When
|
||||
e, err := domain.NewEmail(raw)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Then — round-trip property
|
||||
e2, err := domain.NewEmail(e.String())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, e, e2)
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
`rapid` shrinks failing cases to minimal counterexamples. Use for:
|
||||
|
||||
- Round-trips (parse → serialize → parse).
|
||||
- Algebraic properties (sort produces ordered, dedup is idempotent, JSON marshal/unmarshal is involutive).
|
||||
- Invariants under random input (validator never panics, serializer never produces invalid UTF-8).
|
||||
|
||||
---
|
||||
|
||||
## HTTP testing — `httptest`
|
||||
|
||||
### Server side
|
||||
|
||||
```go
|
||||
func Test_GetUser_returns_user_for_existing_id(t *testing.T) {
|
||||
// Given
|
||||
svc := newSvcWithFake(t)
|
||||
r := gin.New()
|
||||
h := &Handler{Users: svc}
|
||||
h.Mount(r)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/v1/users/u-1", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
// When
|
||||
r.ServeHTTP(rec, req)
|
||||
|
||||
// Then
|
||||
require.Equal(t, 200, rec.Code)
|
||||
var body domain.User
|
||||
require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body))
|
||||
require.Equal(t, "u-1", string(body.ID))
|
||||
}
|
||||
```
|
||||
|
||||
### Client side — `httptest.NewServer`
|
||||
|
||||
```go
|
||||
func Test_Client_retries_on_500(t *testing.T) {
|
||||
// Given — fake upstream
|
||||
var calls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
if calls < 3 {
|
||||
w.WriteHeader(500)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(200)
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client := myclient.New(srv.URL)
|
||||
|
||||
// When
|
||||
err := client.DoSomething(context.Background())
|
||||
|
||||
// Then
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 3, calls)
|
||||
}
|
||||
```
|
||||
|
||||
`httptest.NewServer` spins a real HTTP server on a random port. The fake handler implements the upstream contract. Test the client against the contract, not the implementation.
|
||||
|
||||
---
|
||||
|
||||
## Determinism — the cardinal rules
|
||||
|
||||
- **No `time.Sleep` in tests.** If you need delay, you need a Clock injection.
|
||||
- **`go test -shuffle=on`** in every CI run.
|
||||
- **`go test -count=1`** to defeat the cache.
|
||||
- **Subscribe to the event, do not poll for it.** Channels, callbacks, `t.Cleanup` over polling.
|
||||
- **`t.Parallel()`** for tests that share no state. Speeds up large suites by 4-8x.
|
||||
|
||||
A test that fails 1-in-10 runs is a bug, not flake. The race detector + `-shuffle=on` + ordering hygiene catches >95% of "flake".
|
||||
|
||||
---
|
||||
|
||||
## Benchmarks — `testing.B` + `benchstat`
|
||||
|
||||
```go
|
||||
func Benchmark_NewEmail(b *testing.B) {
|
||||
for b.Loop() { // Go 1.24+ idiom, replaces `for i := 0; i < b.N; i++`
|
||||
_, _ = domain.NewEmail("alice@example.com")
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test -bench=. -count=10 -benchmem ./... | tee bench.txt
|
||||
benchstat bench.txt # statistical comparison
|
||||
```
|
||||
|
||||
Always `-count=10` for stable means. `-benchmem` reports allocations. A 5%-slower benchmark in one run is noise; 10 runs + benchstat tells you what is real.
|
||||
|
||||
To compare before/after a change:
|
||||
|
||||
```bash
|
||||
git stash
|
||||
go test -bench=. -count=10 ./... > before.txt
|
||||
git stash pop
|
||||
go test -bench=. -count=10 ./... > after.txt
|
||||
benchstat before.txt after.txt
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Coverage — the right target
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test -race -shuffle=on -coverprofile=cover.out ./...
|
||||
go tool cover -html=cover.out -o cover.html
|
||||
```
|
||||
|
||||
**Aim for 80%+ on `internal/domain` and `internal/service`.** Boundary code (handlers, store mappers) is exercised by integration tests, where line coverage understates what is actually verified. Do not chase 100% — the last 5% is usually error paths that need fault-injection to hit.
|
||||
|
||||
The `golangci-lint` config does not enforce a minimum — coverage as a CI gate becomes a goal-displacement metric. Treat it as feedback, not requirement.
|
||||
|
||||
---
|
||||
|
||||
## TUI testing — `teatest`
|
||||
|
||||
```go
|
||||
import teatest "charm.land/bubbletea/v2/teatest"
|
||||
|
||||
func Test_Counter_increments_on_space(t *testing.T) {
|
||||
// Given
|
||||
tm := teatest.NewTestModel(t, initial(), teatest.WithInitialTermSize(80, 24))
|
||||
|
||||
// When
|
||||
tm.Send(tea.KeyPressMsg{Code: ' '})
|
||||
|
||||
// Then
|
||||
final := tm.FinalModel(t).(model)
|
||||
require.Equal(t, 1, final.count)
|
||||
}
|
||||
```
|
||||
|
||||
For full-view regression, snapshot the rendered output via `autogold`.
|
||||
|
||||
---
|
||||
|
||||
## Antipatterns the skill rejects
|
||||
|
||||
| Bad | Why | Good |
|
||||
|---|---|---|
|
||||
| `if got != want { t.Errorf("expected %v got %v", want, got) }` | Reinvents `require.Equal` | Use testify |
|
||||
| `time.Sleep(100 * time.Millisecond)` after triggering async work | Flake | Subscribe to completion signal, bounded await |
|
||||
| `t.Skip(...)` to silence a known failure | Buries the bug | Fix or open an issue; never silently skip |
|
||||
| One mega-test asserting 12 things | First failure hides next 11 | Split by `Then` |
|
||||
| Snapshot-everything | Locks formatting, not behavior | Snapshots for structure, asserts for values |
|
||||
| Mock every collaborator | Test asserts implementation, not behavior | Real or fake, never mock everything |
|
||||
| Test calls private function via `_test.go` in same package only | Couples test to implementation | Test through the public surface |
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- testify: https://github.com/stretchr/testify
|
||||
- goleak: https://github.com/uber-go/goleak
|
||||
- autogold: https://github.com/hexops/autogold
|
||||
- rapid: https://pkg.go.dev/pgregory.net/rapid
|
||||
- testcontainers-go: https://golang.testcontainers.org
|
||||
- benchstat: https://pkg.go.dev/golang.org/x/perf/cmd/benchstat
|
||||
- "Go test naming conventions" (Dave Cheney): https://dave.cheney.net/practical-go/presentations/qcon-china.html
|
||||
@@ -0,0 +1,298 @@
|
||||
# Type Patterns
|
||||
|
||||
How to use Go's *limited* type system to catch bugs at compile time. Go gives you fewer tools than Python/TS/Rust — this document covers the four patterns that buy back most of the safety.
|
||||
|
||||
The four patterns:
|
||||
|
||||
1. **Named types** for branding primitives (the Go answer to `NewType` / branded TS).
|
||||
2. **Smart constructors with unexported fields** for parse-don't-validate.
|
||||
3. **Sealed interfaces** for sum types, with `type switch` + `exhaustive` linter.
|
||||
4. **Generics with constraints** for bounded polymorphism (1.18+).
|
||||
|
||||
---
|
||||
|
||||
## 1. Named types — distinct primitives
|
||||
|
||||
Same underlying type, different meaning. The Go type checker prevents *implicit* mixing — but explicit conversion is always possible. Treat this as a contract enforced at boundaries.
|
||||
|
||||
```go
|
||||
package domain
|
||||
|
||||
type UserID string
|
||||
type OrderID string
|
||||
type EmailRaw string // raw, unvalidated string from input
|
||||
|
||||
func GetUser(id UserID) User { /* ... */ }
|
||||
|
||||
uid := UserID("u-123")
|
||||
oid := OrderID("o-456")
|
||||
|
||||
GetUser(uid) // ✅ OK
|
||||
GetUser(oid) // ❌ cannot use oid (type OrderID) as UserID
|
||||
GetUser("u-123") // ❌ untyped string literal — Go DOES catch this
|
||||
GetUser(UserID("u-123")) // ✅ explicit conversion — accept it
|
||||
```
|
||||
|
||||
**Use when**: IDs, opaque tokens, foreign keys, units that share a base primitive.
|
||||
|
||||
**Reality check**: Go does NOT prevent `UserID(orderIDAsString)`. The defense is **smart constructors** for everything beyond an internal identifier. Use named types for cheap brand-only protection; combine with constructors for protection that actually holds.
|
||||
|
||||
### Time-of-day units
|
||||
|
||||
```go
|
||||
type Milliseconds int64
|
||||
type Seconds int64
|
||||
|
||||
func (ms Milliseconds) ToSeconds() Seconds {
|
||||
return Seconds(ms / 1000)
|
||||
}
|
||||
```
|
||||
|
||||
No implicit `Milliseconds + Seconds`. The compiler refuses. Convert explicitly.
|
||||
|
||||
---
|
||||
|
||||
## 2. Smart constructors with unexported fields — the Go answer to Pydantic/Zod
|
||||
|
||||
The single most important pattern in this document. **Go has no Pydantic. It has this.**
|
||||
|
||||
```go
|
||||
package domain
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidEmail = errors.New("invalid email")
|
||||
emailRe = regexp.MustCompile(`^[^@\s]+@[^@\s]+\.[^@\s]+$`)
|
||||
)
|
||||
|
||||
// Email is a parsed, lowercased, valid email address.
|
||||
// The zero value is invalid; construct via NewEmail.
|
||||
type Email struct {
|
||||
raw string // unexported — cannot be set from outside the package
|
||||
}
|
||||
|
||||
func NewEmail(s string) (Email, error) {
|
||||
s = strings.TrimSpace(strings.ToLower(s))
|
||||
if !emailRe.MatchString(s) {
|
||||
return Email{}, ErrInvalidEmail
|
||||
}
|
||||
return Email{raw: s}, nil
|
||||
}
|
||||
|
||||
// String implements fmt.Stringer for printing.
|
||||
func (e Email) String() string { return e.raw }
|
||||
|
||||
// MarshalJSON keeps the wire format unchanged.
|
||||
func (e Email) MarshalJSON() ([]byte, error) {
|
||||
return []byte(`"` + e.raw + `"`), nil
|
||||
}
|
||||
|
||||
// UnmarshalJSON is the parsing boundary — strict mode.
|
||||
func (e *Email) UnmarshalJSON(data []byte) error {
|
||||
if len(data) < 2 || data[0] != '"' || data[len(data)-1] != '"' {
|
||||
return ErrInvalidEmail
|
||||
}
|
||||
parsed, err := NewEmail(string(data[1 : len(data)-1]))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*e = parsed
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
**Why this works**:
|
||||
|
||||
- `Email{raw: "anything"}` from outside the `domain` package is a compile error — `raw` is unexported.
|
||||
- The only way to obtain a non-zero `Email` is `NewEmail(...)`, which validates.
|
||||
- `UnmarshalJSON` routes wire input through the same constructor — boundary parsing is automatic.
|
||||
- Once a function signature has `email Email`, the caller has *proven* it is valid. No internal `if email == ""` checks.
|
||||
|
||||
**Use for every domain value that has invariants**: emails, URLs, phone numbers, currency amounts, percentages, semver versions, IDs with format constraints, time ranges, anything you currently validate in three places.
|
||||
|
||||
### The "zero value problem"
|
||||
|
||||
Go's zero value (`Email{}`) is reachable. The mitigation is documentation + a `IsValid()` method when needed:
|
||||
|
||||
```go
|
||||
func (e Email) IsZero() bool { return e.raw == "" }
|
||||
```
|
||||
|
||||
Or accept it: receivers that take `Email` should *never* receive a zero-value `Email` in correct code. Tests verify it.
|
||||
|
||||
---
|
||||
|
||||
## 3. Sealed interfaces — sum types in Go
|
||||
|
||||
Go has no sum types. The closest thing: an interface with an **unexported method** that only types in the same package can satisfy, dispatched via `type switch`, with the `exhaustive` linter ensuring completeness.
|
||||
|
||||
```go
|
||||
package event
|
||||
|
||||
// Event is a closed sum: Created | Updated | Deleted.
|
||||
// The sealed() method is unexported so external packages cannot add variants.
|
||||
type Event interface {
|
||||
sealed()
|
||||
OccurredAt() time.Time
|
||||
}
|
||||
|
||||
type Created struct {
|
||||
UserID UserID
|
||||
Email Email
|
||||
Timestamp time.Time
|
||||
}
|
||||
func (Created) sealed() {}
|
||||
func (e Created) OccurredAt() time.Time { return e.Timestamp }
|
||||
|
||||
type Updated struct {
|
||||
UserID UserID
|
||||
Changes map[string]any
|
||||
Timestamp time.Time
|
||||
}
|
||||
func (Updated) sealed() {}
|
||||
func (e Updated) OccurredAt() time.Time { return e.Timestamp }
|
||||
|
||||
type Deleted struct {
|
||||
UserID UserID
|
||||
Reason string
|
||||
Timestamp time.Time
|
||||
}
|
||||
func (Deleted) sealed() {}
|
||||
func (e Deleted) OccurredAt() time.Time { return e.Timestamp }
|
||||
```
|
||||
|
||||
Consumer code:
|
||||
|
||||
```go
|
||||
func Render(e event.Event) string {
|
||||
switch v := e.(type) {
|
||||
case event.Created:
|
||||
return fmt.Sprintf("created %s with %s", v.UserID, v.Email)
|
||||
case event.Updated:
|
||||
return fmt.Sprintf("updated %s: %v", v.UserID, v.Changes)
|
||||
case event.Deleted:
|
||||
return fmt.Sprintf("deleted %s (reason: %s)", v.UserID, v.Reason)
|
||||
default:
|
||||
panic(fmt.Sprintf("unhandled event variant: %T", v))
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The `panic` in `default` is the Go equivalent of TS's `assertNever` or Python's `assert_never`. It is only reachable if a new variant is added without updating the switch.
|
||||
|
||||
### The `exhaustive` linter — your compiler
|
||||
|
||||
```yaml
|
||||
# .golangci.yml
|
||||
linters:
|
||||
enable: [exhaustive]
|
||||
linters-settings:
|
||||
exhaustive:
|
||||
check:
|
||||
- switch
|
||||
- map
|
||||
default-signifies-exhaustive: false
|
||||
```
|
||||
|
||||
Now adding `event.Suspended` without updating `Render` is a **lint error**. This is the closest thing Go has to Rust's match exhaustiveness check. **Treat it as compulsory.**
|
||||
|
||||
### Sealed interface gotchas
|
||||
|
||||
- The method MUST be unexported (`sealed()`, not `Sealed()`). Otherwise other packages can implement it.
|
||||
- `type switch` with `*Created` vs `Created` matters — pick value receivers and value cases, or pointer receivers and pointer cases. **Mixing them causes silent miss.**
|
||||
- `interface{}` is not a sealed type. Anything implementing zero methods satisfies it. Sealed interfaces have at least the `sealed()` method.
|
||||
|
||||
---
|
||||
|
||||
## 4. Generics with constraints — bounded polymorphism
|
||||
|
||||
Go 1.18+. Use for genuinely generic algorithms; **do not** use for "I want this to accept anything".
|
||||
|
||||
```go
|
||||
import "cmp"
|
||||
|
||||
// Ordered constraint includes all ordered types (int, float, string, …).
|
||||
func Max[T cmp.Ordered](a, b T) T {
|
||||
if a > b { return a }
|
||||
return b
|
||||
}
|
||||
|
||||
// Custom constraint
|
||||
type Stringer interface {
|
||||
String() string
|
||||
}
|
||||
|
||||
func Join[T Stringer](items []T, sep string) string {
|
||||
parts := make([]string, len(items))
|
||||
for i, item := range items {
|
||||
parts[i] = item.String()
|
||||
}
|
||||
return strings.Join(parts, sep)
|
||||
}
|
||||
```
|
||||
|
||||
The `cmp.Ordered` (Go 1.21+), `cmp.Compare`, and `slices`/`maps` packages cover the common cases without you writing constraints.
|
||||
|
||||
### When NOT to use generics
|
||||
|
||||
- "I want to accept multiple types, so I'll make it generic." Use an **interface** instead. Generics are for parametric polymorphism (same code, different types). Interfaces are for behavioral polymorphism (different code behind a contract).
|
||||
- "I want to return `any`." Use a sealed interface and a `type switch`. `any` returns are anti-patterns past public APIs.
|
||||
|
||||
---
|
||||
|
||||
## 5. Type assertions — the controlled escape hatch
|
||||
|
||||
```go
|
||||
// Bad — panics on failure
|
||||
e := evt.(event.Created)
|
||||
|
||||
// Good — comma-ok form, always
|
||||
if e, ok := evt.(event.Created); ok {
|
||||
// use e
|
||||
}
|
||||
|
||||
// Use errors.As for error chains
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) {
|
||||
// pgErr is the wrapped pg error
|
||||
}
|
||||
```
|
||||
|
||||
**The `errcheck` and `errorlint` linters reject bare type assertions on `error` values.** Use `errors.As`. See `error-handling.md`.
|
||||
|
||||
---
|
||||
|
||||
## 6. Pointers vs values — the only durable rule
|
||||
|
||||
You will see endless debates. The rule that holds up:
|
||||
|
||||
- **If a type has a mutex, never copy it.** Use `*T` everywhere.
|
||||
- **If a type is large (> 64 bytes) and read-only, pass by value or pointer is a measured choice.** Default to pointer for "large" things.
|
||||
- **Receivers must be consistent.** All methods on `T` either take `T` or `*T`. Don't mix. The `staticcheck` linter catches mixed-receiver bugs.
|
||||
- **`nil` pointer = absence. Zero value = "not set yet".** Choose ONE convention per type. Document it.
|
||||
|
||||
---
|
||||
|
||||
## 7. `any` / `interface{}` — when it is acceptable
|
||||
|
||||
Almost never in domain code. Acceptable cases:
|
||||
|
||||
- JSON parsing of genuinely heterogeneous payloads (and even then, prefer `json.RawMessage` + targeted parsing).
|
||||
- `fmt.Sprintf` arguments (variadic `any` is unavoidable here).
|
||||
- Generic container internals before the user-facing API.
|
||||
|
||||
The skill rejects `any` in handler signatures, service signatures, store signatures. If you find yourself writing `func Handle(payload any) error`, you have a sealed-interface waiting to happen.
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- "Parse, don't validate" — Alexis King: https://lexi-lambda.github.io/blog/2019/11/05/parse-don-t-validate/
|
||||
- exhaustive linter: https://github.com/nishanths/exhaustive
|
||||
- Generics constraints: https://go.dev/blog/intro-generics
|
||||
- cmp.Ordered: https://pkg.go.dev/cmp
|
||||
@@ -0,0 +1,314 @@
|
||||
|
||||
# Python Programmer
|
||||
|
||||
Modern Python. Type-strict, stack-first, async-correct.
|
||||
|
||||
## Philosophy
|
||||
|
||||
The type checker is your compiler. Make illegal states unrepresentable. Parse at boundaries. Own resources explicitly. Every function has a contract; the type system enforces it.
|
||||
|
||||
## Hard rules
|
||||
|
||||
These are deliberate project choices. Violations are always wrong, not "style preferences".
|
||||
|
||||
### Tooling
|
||||
|
||||
| Category | Use | Never |
|
||||
|---|---|---|
|
||||
| Package manager | `uv` | pip, poetry, conda, pipenv |
|
||||
| Type checker | `basedpyright` (`typeCheckingMode = "all"`) | pyright, mypy |
|
||||
| Linter + formatter | `ruff` (`select = ["ALL"]`) | flake8, black, isort, autopep8 |
|
||||
| Async runtime | `anyio` | `import asyncio` |
|
||||
| Data | `polars` + `duckdb` + `numpy` | pandas |
|
||||
| Web framework | FastAPI + Pydantic v2 | Flask, Django REST |
|
||||
| ORM | SQLAlchemy 2.x async | Django ORM, Tortoise |
|
||||
| HTTP client | [`httpx2`](https://github.com/pydantic/httpx2) | requests, aiohttp, httpx |
|
||||
| Testing | `pytest` | unittest |
|
||||
| CLI | `typer` + `rich` | argparse, click, fire |
|
||||
|
||||
### The iron list
|
||||
|
||||
1. **Frozen by default** — `@dataclass(frozen=True, slots=True)`. Pydantic: `model_config = ConfigDict(frozen=True)`. Mutable only when mutation is the documented purpose.
|
||||
2. **NewType for distinct IDs** — `UserId = NewType("UserId", int)`. Never pass raw `int` where a branded type exists.
|
||||
3. **`match` only for variants, `if` only for booleans** — **NEVER** use `if/elif/else` to discriminate on type (`isinstance`), enum value, or literal variant. `match/case` is mandatory for these — non-negotiable. **ALWAYS** end with `case unreachable: assert_never(unreachable)` — bare `case _: pass` and `case _: raise ValueError` are banned (they silently swallow new variants). `if/else` is fine only for boolean expressions, range checks, and predicate calls that aren't variant discrimination. See "Why `if/elif` on variants is banned" below for examples.
|
||||
4. **Protocol over ABC** — `typing.Protocol` for interfaces. ABC only when you need shared method implementation.
|
||||
5. **No raw dicts in signatures** — params and returns use `TypedDict`, `dataclass`, or Pydantic model. Internal scratch dicts are fine.
|
||||
6. **Parse, don't validate** — constructors produce typed objects or raise. Never pass unvalidated data deeper into the call stack.
|
||||
7. **Typed errors** — error types are dataclasses or exceptions with typed fields. Never `raise ValueError("something")` with a bare string. Use union returns when the caller is within 1-2 call levels and must handle the outcome (repository → service). Use exceptions when the error should propagate up many layers to a boundary handler (service → HTTP handler).
|
||||
8. **Final for constants** — module-level constants use `Final`. Mutable module globals are a code smell.
|
||||
9. **Explicit None** — annotate `-> X | None`. Never return `None` from a function whose signature omits it.
|
||||
10. **Context managers for resources** — files, DB connections, HTTP clients, locks. No manual `.close()`.
|
||||
11. **No Any, no object** — both are banned as type annotations. `object` erases all structural information (zero callable attributes, zero narrowing). Use `Protocol` (structural typing), `TypeVar` (generic pass-through), explicit union (known variants), or `TypedDict` (dict shapes).
|
||||
12. **No cast** — `cast()` is banned. Redesign the types.
|
||||
13. **No type: ignore** — fix the type error. The checker is right; you are wrong.
|
||||
14. **No broad except** — `except Exception` and `except BaseException` are banned. Catch the **specific** exception you expect. A broad catch swallows bugs you need to see — `KeyError`, `AttributeError`, `TypeError` all vanish silently. If you genuinely need a catch-all at a top-level boundary (CLI entry, HTTP handler), use `# noqa: BROAD_EXCEPT_OK` and log + re-raise.
|
||||
|
||||
### Typing and safety
|
||||
|
||||
- `basedpyright` in `typeCheckingMode = "all"`. Every public function has full annotations. Internal helpers: annotate return type; parameter types may be inferred.
|
||||
- `ruff` with `select = ["ALL"]`. Override specific rules per project in `pyproject.toml`, never globally disable the strict baseline.
|
||||
- Every new function must have a `docstring` unless its name + signature makes it completely obvious (e.g. `def full_name(first: str, last: str) -> str:`).
|
||||
- Use `X | Y` union syntax (PEP 604), never `Union[X, Y]` or `Optional[X]`.
|
||||
|
||||
### Why `object` is banned
|
||||
|
||||
`object` pretends to be safe ("it's the top type!") but gives **zero** narrowing and **zero** attributes. Even `Any` is more honest — it admits the boundary is untyped.
|
||||
|
||||
```python
|
||||
# BANNED
|
||||
def process(data: object) -> object: ...
|
||||
def store(items: list[object]) -> None: ...
|
||||
results: dict[str, object] = {}
|
||||
|
||||
# GOOD — Protocol for structural typing
|
||||
class Serializable(Protocol):
|
||||
def serialize(self) -> bytes: ...
|
||||
def process(data: Serializable) -> ProcessResult: ...
|
||||
|
||||
# GOOD — TypeVar for generic pass-through
|
||||
def identity[T](x: T) -> T: ...
|
||||
def first[T](items: Sequence[T]) -> T: ...
|
||||
|
||||
# GOOD — explicit union for known variants
|
||||
def parse(raw: str | bytes) -> Document: ...
|
||||
```
|
||||
|
||||
### Why `if/elif` on variants is banned
|
||||
|
||||
`if/elif/else` chains on type, enum, or literal values lose compile-time exhaustiveness. When a new variant is added, nothing warns you. `match/case` + `assert_never` does.
|
||||
|
||||
```python
|
||||
# BANNED — if/elif for type discrimination
|
||||
if isinstance(event, Click):
|
||||
handle_click(event.x, event.y)
|
||||
elif isinstance(event, Scroll):
|
||||
handle_scroll(event.delta)
|
||||
else:
|
||||
raise ValueError(f"Unknown: {event}") # runtime bomb
|
||||
|
||||
# BANNED — if/elif for enum discrimination
|
||||
if status == Status.PENDING:
|
||||
start_review()
|
||||
elif status == Status.ACTIVE:
|
||||
continue_processing()
|
||||
elif status == Status.CLOSED:
|
||||
archive()
|
||||
|
||||
# BANNED — non-exhaustive match (swallows new variants)
|
||||
match event:
|
||||
case Click(x, y): handle_click(x, y)
|
||||
case _: pass
|
||||
|
||||
# GOOD — exhaustive match with assert_never
|
||||
match event:
|
||||
case Click(x=x, y=y):
|
||||
handle_click(x, y)
|
||||
case Scroll(delta=delta):
|
||||
handle_scroll(delta)
|
||||
case unreachable:
|
||||
assert_never(unreachable)
|
||||
|
||||
# GOOD — enum match
|
||||
match status:
|
||||
case Status.PENDING: start_review()
|
||||
case Status.ACTIVE: continue_processing()
|
||||
case Status.CLOSED: archive()
|
||||
case unreachable: assert_never(unreachable)
|
||||
```
|
||||
|
||||
`if/else` is fine for boolean conditions and range checks — things that aren't variant discrimination:
|
||||
|
||||
```python
|
||||
# FINE — boolean, not variant
|
||||
if age >= 18:
|
||||
grant_access()
|
||||
else:
|
||||
deny_access()
|
||||
```
|
||||
|
||||
### Why broad `except` is banned
|
||||
|
||||
`except Exception` catches **every** non-system exception — `KeyError`, `TypeError`, `AttributeError`, `ValueError` all vanish. You lose the stack trace that would have told you exactly what went wrong. The fix is always to name the exception you expect.
|
||||
|
||||
```python
|
||||
# BANNED — swallows bugs
|
||||
try:
|
||||
result = api.fetch(url)
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
return None
|
||||
|
||||
# BANNED — catch-and-ignore
|
||||
try:
|
||||
parse(data)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# GOOD — catch what you expect
|
||||
try:
|
||||
result = api.fetch(url)
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.error("API %d: %s", e.response.status_code, e.request.url)
|
||||
return None
|
||||
except httpx.ConnectError:
|
||||
raise ServiceUnavailableError(service="api") from None
|
||||
|
||||
# GOOD — top-level boundary (only place broad catch is acceptable)
|
||||
def main() -> int: # noqa: BROAD_EXCEPT_OK
|
||||
try:
|
||||
return run()
|
||||
except Exception:
|
||||
logger.exception("unhandled error")
|
||||
return 1
|
||||
```
|
||||
|
||||
### Async
|
||||
|
||||
- `import asyncio` is **BANNED**. Use `import anyio`.
|
||||
- For background tasks, use `anyio.create_task_group`. Never fire-and-forget with `asyncio.create_task`.
|
||||
- For concurrency gates, use `anyio.CapacityLimiter` (not `asyncio.Semaphore`).
|
||||
- Load `async-anyio.md` when writing async code for the full pattern library.
|
||||
|
||||
### Data modeling — which container, when
|
||||
|
||||
All model fields carry type annotations. No `Any`, no untyped dicts in public APIs.
|
||||
Use `polars` + `duckdb` for data. pandas is never the right answer in this stack.
|
||||
|
||||
| Situation | Use |
|
||||
|---|---|
|
||||
| User input, API request/response | `Pydantic BaseModel (frozen=True)` |
|
||||
| Internal value object (no I/O) | `@dataclass(frozen=True, slots=True)` |
|
||||
| Function with multiple outcomes | Union of frozen dataclasses + `match` |
|
||||
| Dict shape for JSON compat / `**kwargs` | `TypedDict` |
|
||||
| Fixed constants | `StrEnum` / `IntEnum` |
|
||||
| Distinct primitive (UserId vs MovieId) | `NewType` |
|
||||
| Contract / capability | `Protocol` |
|
||||
| Contract + shared implementation | `ABC` |
|
||||
| ORM model (SQLAlchemy) | `Mapped[]` — inherently mutable, `# noqa: MUTABLE_OK` |
|
||||
| Config from env vars | `pydantic-settings BaseSettings` |
|
||||
|
||||
**The one rule**: data crosses trust boundary → Pydantic. Everything else → dataclass.
|
||||
|
||||
Load `data-modeling.md` for the full decision flowchart and comparison matrix.
|
||||
|
||||
### When frozen=True does not apply
|
||||
|
||||
- **ORM models** — SQLAlchemy `Mapped[]` requires mutation. Use `# noqa: MUTABLE_OK`.
|
||||
- **Builder / accumulator** — object exists to be mutated (counter, buffer, state machine). Docstring must explain why.
|
||||
- **Pydantic Settings** — tests override fields. Mutable is acceptable.
|
||||
|
||||
If you need `# noqa: MUTABLE_OK`, the class docstring must say why mutation is required.
|
||||
|
||||
### Libraries
|
||||
|
||||
Canonical defaults (override only if `pyproject.toml` explicitly picks something else):
|
||||
|
||||
| Domain | Library | Reason |
|
||||
|---|---|---|
|
||||
| CLI | `typer` | Type-annotated CLI from function sigs |
|
||||
| Pretty output | `rich` | Tables, progress, tracebacks, markdown |
|
||||
| HTTP client | [`httpx2`](https://github.com/pydantic/httpx2) | Next-gen HTTP client (Pydantic stewardship), HTTP/2, brotli+zstd. Always `httpx2[http2,brotli,zstd]`. See `httpx2-optimization.md` |
|
||||
| Validation | `pydantic` v2 | Fast native validator, JSON Schema |
|
||||
| Web API | `fastapi` | Async, Pydantic-native, OpenAPI |
|
||||
| ORM | `sqlalchemy` 2.x async | `Mapped[]` types, async sessions |
|
||||
| DB driver (Postgres) | `asyncpg` (via SQLAlchemy) | Fastest PG driver |
|
||||
| AI agents | `pydantic-ai` | Typed deps, structured output |
|
||||
| TUI | `textual` | Rich-based, CSS layout, widgets |
|
||||
| Logging | `rich.logging.RichHandler` | Pretty; swap to `structlog` in prod |
|
||||
|
||||
## pyproject.toml — the one true config
|
||||
|
||||
Scaffold a new project with all strict defaults pre-configured:
|
||||
|
||||
```bash
|
||||
uv run ../../scripts/python/new-project.py myproject
|
||||
uv run ../../scripts/python/new-project.py myproject --path ./workspace
|
||||
uv run ../../scripts/python/new-project.py myproject --lib # publishable library
|
||||
```
|
||||
|
||||
Creates via `uv init`, then injects basedpyright `typeCheckingMode = "all"` + ruff `select = ["ALL"]` + pytest strict. Cross-platform (macOS, Linux, Windows).
|
||||
|
||||
For manual setup: `uv init --app myproject`, then load `pyproject-strict.md`.
|
||||
|
||||
## PEP 723 — inline script metadata (mandatory for ALL scripts)
|
||||
|
||||
Every `.py` script — even throwaway — MUST use PEP 723 inline metadata with the `# ─── How to run ───` comment block. No venv, no `requirements.txt`. The script IS the environment spec. A script without the usage comment block is incomplete.
|
||||
|
||||
Scaffold with: `uv run ../../scripts/python/new-script.py <name> --deps "httpx2[http2,brotli,zstd]"` (writes to temp dir by default, `--output` for specific path).
|
||||
|
||||
Load `one-liners.md` for full patterns, examples, and anti-patterns.
|
||||
|
||||
## Reference loading
|
||||
|
||||
Load on demand — not all at once.
|
||||
|
||||
| Need | Load |
|
||||
|---|---|
|
||||
| Full pyproject.toml config | `pyproject-strict.md` |
|
||||
| Type patterns (NewType, Final, enums, narrowing) | `type-patterns.md` |
|
||||
| Data modeling (container choice, frozen, parse-don't-validate) | `data-modeling.md` |
|
||||
| Error handling (typed errors, union returns, exhaustive match) | `error-handling.md` |
|
||||
| Async patterns (anyio) | `async-anyio.md` |
|
||||
| Data processing (polars / duckdb) | `data-processing.md` |
|
||||
| FastAPI + SQLAlchemy stack | `fastapi-stack.md` |
|
||||
| Library decision tree | `libraries.md` |
|
||||
| **httpx2 optimization** (MUST load for any network code) | `httpx2-optimization.md` |
|
||||
| **orjson** (when JSON is in the hot path; FastAPI/Pydantic v2 integration) | `orjson-stack.md` |
|
||||
| One-liner scripts (PEP 723) | `one-liners.md` |
|
||||
| PydanticAI agents | `pydantic-ai.md` |
|
||||
| Textual TUI | `textual-tui.md` |
|
||||
|
||||
## httpx2 — mandatory for ALL network requests
|
||||
|
||||
Every outgoing HTTP call MUST use [`httpx2`](https://github.com/pydantic/httpx2) (`httpx2[http2,brotli,zstd]`). Never `requests`, never `aiohttp`, never the original `httpx`.
|
||||
|
||||
**ALL optimizations are ON by default — not optional, not progressive, not "nice to have".** A bare `httpx2.AsyncClient()` is a bug — treat it like a lint violation. The correct way is the factory pattern in `httpx2-optimization.md` with: HTTP/2 enabled, tuned connection pool (200/40/30s), split timeouts (5/30/10/10), transport retries (3), TCP_NODELAY, follow_redirects, and event hooks for observability.
|
||||
|
||||
When writing or reviewing ANY network code, **ALWAYS load `httpx2-optimization.md`** and use the factory pattern verbatim. No exceptions.
|
||||
|
||||
## No-excuse audit
|
||||
|
||||
Violations caught by `../../scripts/python/check-no-excuse-rules.py`. Run after every edit session.
|
||||
|
||||
| Rule ID | Catches | Opt-out |
|
||||
|---|---|---|
|
||||
| `cast-any` | `cast(Any, ...)` | None — redesign types |
|
||||
| `type-ignore` | `# type: ignore` | None — fix the type |
|
||||
| `pyright-ignore` | `# pyright: ignore` | None — fix the type |
|
||||
| `bare-except` | `except:` with no class | None — name the exception |
|
||||
| `silent-except` | `except X: pass` / `except X: ...` | None — handle or re-raise |
|
||||
| `no-asyncio` | `import asyncio` | `# noqa: ANYIO_OK` |
|
||||
| `no-pandas` | `import pandas` | `# noqa: PANDAS_OK` |
|
||||
| `mutable-dataclass` | `@dataclass` without `frozen=True` | `# noqa: MUTABLE_OK` |
|
||||
| `missing-slots` | `@dataclass` without `slots=True` | `# noqa: SLOTS_OK` |
|
||||
| `raw-dict-return` | `-> dict` in function return type | `# noqa: DICT_OK` |
|
||||
| `missing-assert-never` | `match` block without `assert_never` default | `# noqa: MATCH_OK` |
|
||||
| `generic-exception` | `raise ValueError("...")` / `raise TypeError("...")` with bare string | `# noqa: GENERIC_ERR_OK` |
|
||||
| `no-object` | `object` used as type annotation (param, return, generic arg) | `# noqa: OBJECT_OK` |
|
||||
| `if-elif-on-variant` | `if isinstance()`/`if x == Enum.V` chain that should be `match/case` | `# noqa: IF_VARIANT_OK` |
|
||||
| `oversized-module` | File exceeds 250 pure LOC (non-blank, non-comment) | `# noqa: SIZE_OK` |
|
||||
| `broad-except` | `except Exception` / `except BaseException` (too broad) | `# noqa: BROAD_EXCEPT_OK` |
|
||||
|
||||
Fix every violation before declaring work done. basedpyright + ruff strict config catches the rest.
|
||||
|
||||
## In tests
|
||||
|
||||
Tests are strict too, with these exceptions (already configured in `pyproject.toml` per-file-ignores):
|
||||
|
||||
| In tests you may | Why |
|
||||
|---|---|
|
||||
| Use `assert` | That's how pytest works (`S101` ignored) |
|
||||
| Use magic numbers | Test data (`PLR2004` ignored) |
|
||||
| Access `_private` members | Testing internals (`SLF001` ignored) |
|
||||
| Skip docstrings | Test names are the docs (`D` ignored) |
|
||||
| Have unused function args | Fixtures (`ARG` ignored) |
|
||||
|
||||
Tests still follow the iron list — frozen dataclasses, typed errors, exhaustive match. If test fixtures need mutable state, use `# noqa: MUTABLE_OK` on the fixture class.
|
||||
|
||||
## Existing codebases
|
||||
|
||||
When editing an existing file that doesn't follow these rules: **write new code in strict style, don't refactor existing code in the same change.** Mixing feature work with style migration makes reviews harder and bugs likelier.
|
||||
|
||||
## Activation
|
||||
|
||||
This skill activates whenever you are writing or modifying any `.py` file. Even one-off scripts get the strict treatment — that is the whole point of PEP 723 + uv: production hygiene with throwaway ergonomics.
|
||||
@@ -0,0 +1,442 @@
|
||||
# AnyIO Reference: Replacing asyncio Idioms
|
||||
|
||||
> **Skill mandate**: `import asyncio` is BANNED. Use `import anyio` exclusively.
|
||||
> This reference targets AnyIO 4.x (2026 Python projects).
|
||||
|
||||
---
|
||||
|
||||
## 1. Task Groups (The Core Primitive)
|
||||
|
||||
AnyIO uses **structured concurrency** via task groups. A task group is an async context manager that guarantees all child tasks finish before the block exits.
|
||||
|
||||
### `start_soon` — fire-and-forget
|
||||
|
||||
```python
|
||||
import anyio
|
||||
|
||||
async def worker(n: int) -> None:
|
||||
await anyio.sleep(1)
|
||||
print(f"task {n} done")
|
||||
|
||||
async def main() -> None:
|
||||
async with anyio.create_task_group() as tg:
|
||||
for i in range(3):
|
||||
tg.start_soon(worker, i)
|
||||
print("all tasks finished")
|
||||
|
||||
anyio.run(main)
|
||||
```
|
||||
|
||||
**Signature**: `tg.start_soon(func, *args, name=None)`
|
||||
- `func` must be a **coroutine function** (not a coroutine object).
|
||||
- `name` is optional, for introspection/debugging.
|
||||
- No return value; exceptions propagate as `ExceptionGroup` on exit.
|
||||
|
||||
### `start` — wait for ready signal
|
||||
|
||||
Use when a task must initialize before the caller proceeds (e.g., starting a server and then connecting to it).
|
||||
|
||||
```python
|
||||
from anyio import TASK_STATUS_IGNORED, create_task_group, run
|
||||
from anyio.abc import TaskStatus
|
||||
|
||||
async def start_server(port: int, *, task_status: TaskStatus[None] = TASK_STATUS_IGNORED) -> None:
|
||||
listener = await anyio.create_tcp_listener(local_host="127.0.0.1", local_port=port)
|
||||
task_status.started() # unblocks tg.start()
|
||||
await listener.serve(handler)
|
||||
|
||||
async def main() -> None:
|
||||
async with create_task_group() as tg:
|
||||
await tg.start(start_server, 8080) # blocks until task_status.started()
|
||||
# server is guaranteed ready here
|
||||
async with await anyio.connect_tcp("127.0.0.1", 8080) as client:
|
||||
...
|
||||
|
||||
run(main)
|
||||
```
|
||||
|
||||
**Rule of thumb**:
|
||||
- Use `start_soon` when you don't need to know when the task is ready.
|
||||
- Use `start` when the task must signal readiness before you continue.
|
||||
|
||||
### `create_task` — retrieving return values (AnyIO 4.14+)
|
||||
|
||||
```python
|
||||
async def add(x: int, y: int) -> int:
|
||||
return x + y
|
||||
|
||||
async def main() -> None:
|
||||
async with anyio.create_task_group() as tg:
|
||||
handle = tg.create_task(add(2, 4))
|
||||
result = await handle # == 6
|
||||
print(handle.return_value) # also 6
|
||||
|
||||
anyio.run(main)
|
||||
```
|
||||
|
||||
**Signature**: `tg.create_task(coro, *, name=None, context=None) -> TaskHandle[T]`
|
||||
- Returns a `TaskHandle` you can `await` for the result.
|
||||
- If the task raises, awaiting raises `TaskFailed` (or `TaskCancelled`).
|
||||
- This is the canonical replacement for `asyncio.gather` when you need results.
|
||||
|
||||
---
|
||||
|
||||
## 2. asyncio → anyio Cheat Sheet
|
||||
|
||||
| asyncio | anyio | Notes |
|
||||
|---------|-------|-------|
|
||||
| `asyncio.gather(a, b, c)` | `tg.create_task(a); tg.create_task(b); tg.create_task(c); results = [await h for h in handles]` | No direct gather; structured concurrency requires explicit task group scope. For fire-and-forget, use `tg.start_soon`. |
|
||||
| `asyncio.create_task(coro)` | `tg.start_soon(func, *args)` or `tg.create_task(coro)` | `start_soon` takes a coroutine **function** + args. `create_task` takes a coroutine **object** and returns a handle. |
|
||||
| `asyncio.sleep(n)` | `anyio.sleep(n)` | Identical semantics. |
|
||||
| `asyncio.wait_for(coro, timeout)` | `with anyio.fail_after(timeout): await coro` | Raises `TimeoutError`. Use `move_on_after` for silent timeout. |
|
||||
| `asyncio.Event()` | `anyio.Event()` | AnyIO events are **not reusable**; create a new one instead of `.clear()`. |
|
||||
| `asyncio.Lock()` | `anyio.Lock()` | Use `async with lock:`. Pass `fast_acquire=True` if performance-critical. |
|
||||
| `asyncio.Semaphore(n)` | `anyio.Semaphore(n)` | Same. Pass `fast_acquire=True` if performance-critical. |
|
||||
| `asyncio.Condition()` | `anyio.Condition()` | Same semantics. |
|
||||
| `asyncio.run(main())` | `anyio.run(main)` | Backend-agnostic entry point. |
|
||||
| `asyncio.Queue(maxsize=N)` | `anyio.create_memory_object_stream[T](max_buffer_size=N)` | Returns `(send_stream, receive_stream)`. Supports `async for` on receive end. |
|
||||
| `asyncio.to_thread(fn, *args)` | `anyio.to_thread.run_sync(fn, *args)` | Supports `abandon_on_cancel=True` and custom `limiter`. |
|
||||
| `asyncio.run_coroutine_threadsafe(coro, loop)` | `anyio.from_thread.run(func, *args)` | Call async code from a worker thread. |
|
||||
| `loop.call_soon_threadsafe(callback)` | `anyio.from_thread.run_sync(func, *args)` | Call sync code in event loop thread from worker thread, **with return value**. |
|
||||
| `asyncio.shield(coro)` | `with anyio.CancelScope(shield=True): ...` | AnyIO shielding does not orphan tasks. |
|
||||
| `asyncio.timeout(delay)` | `with anyio.fail_after(delay): ...` | AnyIO uses level cancellation, not edge cancellation. |
|
||||
| `asyncio.CancelledError` | `anyio.get_cancelled_exc_class()` | Use this to catch cancellation portably across backends. |
|
||||
|
||||
---
|
||||
|
||||
## 3. Cancellation & CancelScope
|
||||
|
||||
AnyIO uses **level cancellation** (inspired by Trio), not asyncio's **edge cancellation**.
|
||||
|
||||
- **Edge cancellation** (asyncio): A `CancelledError` is injected once. If caught and not re-raised, the task keeps running.
|
||||
- **Level cancellation** (anyio): As long as a task is inside an effectively cancelled scope, every yield point raises a new cancellation exception.
|
||||
|
||||
### Basic CancelScope
|
||||
|
||||
```python
|
||||
from anyio import CancelScope, create_task_group, get_cancelled_exc_class, sleep, run
|
||||
|
||||
async def worker() -> None:
|
||||
try:
|
||||
await sleep(10)
|
||||
except get_cancelled_exc_class():
|
||||
print("cancelled!")
|
||||
raise # ALWAYS re-raise cancellation exceptions
|
||||
|
||||
async def main() -> None:
|
||||
async with create_task_group() as tg:
|
||||
tg.start_soon(worker)
|
||||
await sleep(0.1)
|
||||
tg.cancel_scope.cancel() # cancels all children
|
||||
|
||||
run(main)
|
||||
```
|
||||
|
||||
### Shielding
|
||||
|
||||
Shield a block from external cancellation. Essential for cleanup.
|
||||
|
||||
```python
|
||||
from anyio import CancelScope, create_task_group, sleep, run
|
||||
|
||||
async def main() -> None:
|
||||
async with create_task_group() as tg:
|
||||
with CancelScope(shield=True):
|
||||
tg.start_soon(some_task)
|
||||
tg.cancel_scope.cancel() # shielded block is protected
|
||||
await sleep(1) # this still runs
|
||||
|
||||
run(main)
|
||||
```
|
||||
|
||||
**Combine with timeouts for graceful shutdown**:
|
||||
|
||||
```python
|
||||
from anyio import CancelScope, move_on_after
|
||||
|
||||
async def do_something(resource) -> None:
|
||||
try:
|
||||
await run_async_stuff()
|
||||
except BaseException:
|
||||
# Allow up to 10s for cleanup, then move on
|
||||
with move_on_after(10, shield=True):
|
||||
await resource.aclose()
|
||||
raise
|
||||
```
|
||||
|
||||
### Structured Concurrency Guarantee
|
||||
|
||||
A task group contains its own `CancelScope`. If any child task raises an exception:
|
||||
1. The task group's cancel scope is cancelled.
|
||||
2. All other child tasks receive cancellation.
|
||||
3. The task group waits for all children to finish.
|
||||
4. The original exception (wrapped in `ExceptionGroup` if multiple) is re-raised.
|
||||
|
||||
---
|
||||
|
||||
## 4. Timeouts
|
||||
|
||||
Two context managers. Both create a `CancelScope` internally.
|
||||
|
||||
### `fail_after` — raises on timeout
|
||||
|
||||
```python
|
||||
from anyio import fail_after, sleep, run
|
||||
|
||||
async def main() -> None:
|
||||
try:
|
||||
with fail_after(5) as scope:
|
||||
await sleep(10)
|
||||
except TimeoutError:
|
||||
print("timed out")
|
||||
print(scope.cancelled_caught) # True
|
||||
|
||||
run(main)
|
||||
```
|
||||
|
||||
### `move_on_after` — silent timeout
|
||||
|
||||
```python
|
||||
from anyio import move_on_after, sleep, run
|
||||
|
||||
async def main() -> None:
|
||||
with move_on_after(5) as scope:
|
||||
await sleep(10)
|
||||
print("this never prints")
|
||||
|
||||
print("exited scope, cancelled =", scope.cancelled_caught)
|
||||
|
||||
run(main)
|
||||
```
|
||||
|
||||
### Combined with shielding
|
||||
|
||||
```python
|
||||
from anyio import move_on_after
|
||||
|
||||
# Give cleanup 10 seconds, but don't let outer cancellation interrupt it
|
||||
with move_on_after(10, shield=True):
|
||||
await resource.aclose()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Memory Object Streams (Queue Replacement)
|
||||
|
||||
Replaces `asyncio.Queue` with a safer, typed, structured-concurrency-friendly construct.
|
||||
|
||||
```python
|
||||
from anyio import create_task_group, create_memory_object_stream, run
|
||||
from anyio.streams.memory import MemoryObjectReceiveStream
|
||||
|
||||
async def consumer(stream: MemoryObjectReceiveStream[str]) -> None:
|
||||
async with stream: # closes receive end on exit
|
||||
async for item in stream:
|
||||
print("received", item)
|
||||
|
||||
async def main() -> None:
|
||||
# Type-annotated stream creation (AnyIO 4+ syntax)
|
||||
send_stream, receive_stream = create_memory_object_stream[str](max_buffer_size=10)
|
||||
|
||||
async with create_task_group() as tg:
|
||||
tg.start_soon(consumer, receive_stream)
|
||||
async with send_stream:
|
||||
for i in range(5):
|
||||
await send_stream.send(f"item {i}")
|
||||
# send_stream closed → consumer's async for loop exits naturally
|
||||
|
||||
run(main)
|
||||
```
|
||||
|
||||
**Key differences from `asyncio.Queue`**:
|
||||
- **Bounded by default**: `max_buffer_size=0` means send blocks until a receiver is ready.
|
||||
- **Cloneable**: Each producer/consumer can close its own clone. The stream only ends when **all** clones of one end are closed.
|
||||
- **Async iterable**: `async for item in receive_stream:` works out of the box.
|
||||
- **Type-safe**: Generic `create_memory_object_stream[T]()`.
|
||||
- **Synchronous close**: Both `close()` and `async with` work.
|
||||
|
||||
---
|
||||
|
||||
## 6. Backend Selection
|
||||
|
||||
AnyIO is backend-agnostic. Code written against AnyIO APIs runs on both asyncio and Trio.
|
||||
|
||||
```python
|
||||
import anyio
|
||||
|
||||
async def main() -> None:
|
||||
print("running on", anyio.current_async_library())
|
||||
await anyio.sleep(1)
|
||||
|
||||
# Default backend (asyncio)
|
||||
anyio.run(main)
|
||||
|
||||
# Explicit backend
|
||||
anyio.run(main, backend="trio")
|
||||
anyio.run(main, backend="asyncio", backend_options={"debug": True})
|
||||
```
|
||||
|
||||
**Library design rule**: Never hardcode a backend. Let the application choose via `anyio.run()`. Libraries should only import `anyio` and avoid backend-specific APIs.
|
||||
|
||||
---
|
||||
|
||||
## 7. Compatibility with asyncio-only libraries
|
||||
|
||||
### Using asyncio libraries under the asyncio backend
|
||||
|
||||
If a third-party library exposes only an asyncio interface (returns asyncio coroutine objects), it works directly under the asyncio backend because AnyIO runs on top of asyncio's event loop:
|
||||
|
||||
```python
|
||||
import anyio
|
||||
import some_asyncio_only_lib # returns asyncio.Future/coroutine objects
|
||||
|
||||
async def main() -> None:
|
||||
# This works because under the asyncio backend, await passes through
|
||||
result = await some_asyncio_only_lib.fetch_data()
|
||||
|
||||
anyio.run(main, backend="asyncio")
|
||||
```
|
||||
|
||||
**Important**: This only works on the `asyncio` backend. On the `trio` backend, asyncio-native objects will not work.
|
||||
|
||||
### When you MUST use asyncio APIs
|
||||
|
||||
Some APIs have no AnyIO equivalent and require direct event loop access:
|
||||
|
||||
| Scenario | asyncio API | AnyIO approach |
|
||||
|----------|-------------|----------------|
|
||||
| Signal handlers | `loop.add_signal_handler()` | `anyio.open_signal_receiver()` |
|
||||
| Custom protocols | `asyncio.Protocol` | Use AnyIO streams / sockets |
|
||||
| Direct Future manipulation | `asyncio.Future` | Avoid; use AnyIO primitives |
|
||||
| Eager task factories | `asyncio.eager_task_factory` | Experimental in AnyIO; avoid |
|
||||
|
||||
If you absolutely need the running loop:
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
|
||||
async def main() -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
# ... do something loop-specific ...
|
||||
# WARNING: this breaks backend-agnosticism
|
||||
|
||||
anyio.run(main, backend="asyncio")
|
||||
```
|
||||
|
||||
**Best practice**: Wrap asyncio-only code in a backend-agnostic facade, and document that the feature requires the asyncio backend.
|
||||
|
||||
---
|
||||
|
||||
## 8. Idiomatic Code Snippets
|
||||
|
||||
### Snippet 1: Parallel HTTP requests with timeout and cleanup
|
||||
|
||||
```python
|
||||
import anyio
|
||||
|
||||
async def fetch(url: str) -> bytes:
|
||||
await anyio.sleep(0.5) # simulate
|
||||
return b"data"
|
||||
|
||||
async def main() -> None:
|
||||
urls = ["a", "b", "c"]
|
||||
async with anyio.create_task_group() as tg:
|
||||
with anyio.move_on_after(5):
|
||||
for url in urls:
|
||||
tg.start_soon(fetch, url)
|
||||
# All tasks are cancelled on timeout; task group waits for cleanup
|
||||
|
||||
anyio.run(main)
|
||||
```
|
||||
|
||||
### Snippet 2: Producer-consumer with memory object stream
|
||||
|
||||
```python
|
||||
import anyio
|
||||
from anyio.streams.memory import MemoryObjectReceiveStream
|
||||
|
||||
async def producer(send_stream: anyio.streams.memory.MemoryObjectSendStream[int]) -> None:
|
||||
async with send_stream:
|
||||
for i in range(100):
|
||||
await send_stream.send(i)
|
||||
|
||||
async def consumer(receive_stream: MemoryObjectReceiveStream[int]) -> None:
|
||||
async with receive_stream:
|
||||
async for item in receive_stream:
|
||||
print(f"consumed {item}")
|
||||
|
||||
async def main() -> None:
|
||||
send, receive = anyio.create_memory_object_stream[int](max_buffer_size=5)
|
||||
async with anyio.create_task_group() as tg:
|
||||
tg.start_soon(producer, send)
|
||||
tg.start_soon(consumer, receive)
|
||||
|
||||
anyio.run(main)
|
||||
```
|
||||
|
||||
### Snippet 3: Calling sync code from async
|
||||
|
||||
```python
|
||||
import time
|
||||
import anyio
|
||||
|
||||
async def main() -> None:
|
||||
# Run blocking function in worker thread
|
||||
result = await anyio.to_thread.run_sync(time.sleep, 2)
|
||||
print("done")
|
||||
|
||||
anyio.run(main)
|
||||
```
|
||||
|
||||
### Snippet 4: Calling async code from a worker thread
|
||||
|
||||
```python
|
||||
import anyio
|
||||
|
||||
def blocking_callback() -> None:
|
||||
# Inside a worker thread, call back into the event loop
|
||||
anyio.from_thread.run(anyio.sleep, 1)
|
||||
anyio.from_thread.run_sync(print, "hello from thread")
|
||||
|
||||
async def main() -> None:
|
||||
await anyio.to_thread.run_sync(blocking_callback)
|
||||
|
||||
anyio.run(main)
|
||||
```
|
||||
|
||||
### Snippet 5: Graceful shutdown with shielded cleanup
|
||||
|
||||
```python
|
||||
import anyio
|
||||
|
||||
async def worker() -> None:
|
||||
try:
|
||||
await anyio.sleep_forever()
|
||||
except anyio.get_cancelled_exc_class():
|
||||
with anyio.CancelScope(shield=True):
|
||||
await anyio.sleep(0.5) # cleanup
|
||||
print("cleaned up")
|
||||
raise
|
||||
|
||||
async def main() -> None:
|
||||
async with anyio.create_task_group() as tg:
|
||||
tg.start_soon(worker)
|
||||
await anyio.sleep(1)
|
||||
tg.cancel_scope.cancel()
|
||||
|
||||
anyio.run(main)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- AnyIO Documentation (stable): https://anyio.readthedocs.io/en/stable/
|
||||
- AnyIO GitHub (HEAD `cb245dba`): https://github.com/agronholm/anyio
|
||||
- Task Groups: https://anyio.readthedocs.io/en/stable/tasks.html
|
||||
- Cancellation & Timeouts: https://anyio.readthedocs.io/en/stable/cancellation.html
|
||||
- Streams: https://anyio.readthedocs.io/en/stable/streams.html
|
||||
- Synchronization: https://anyio.readthedocs.io/en/stable/synchronization.html
|
||||
- Threads: https://anyio.readthedocs.io/en/stable/threads.html
|
||||
- Basics / Backends: https://anyio.readthedocs.io/en/stable/basics.html
|
||||
- Design Rationale (why asyncio is problematic): https://anyio.readthedocs.io/en/stable/why.html
|
||||
@@ -0,0 +1,233 @@
|
||||
# Data Modeling
|
||||
|
||||
Which container to use, how to structure data, and why frozen is the default.
|
||||
|
||||
---
|
||||
|
||||
## Decision flowchart
|
||||
|
||||
```
|
||||
Is it a fixed set of named constants?
|
||||
YES → StrEnum / IntEnum
|
||||
NO ↓
|
||||
Is it just branding a primitive (int, str, float)?
|
||||
YES → NewType("X", base)
|
||||
NO ↓
|
||||
Is it an interface / contract ("this thing can do X")?
|
||||
├─ Shape only, no shared code → Protocol
|
||||
└─ Shared method implementation needed → ABC
|
||||
NO ↓
|
||||
Does the data cross a trust boundary (user input, API, file, external DB)?
|
||||
YES → pydantic.BaseModel (frozen=True) — validates + serializes
|
||||
NO ↓
|
||||
Is it a dict shape needed for JSON compat / **kwargs typing?
|
||||
YES → TypedDict
|
||||
NO ↓
|
||||
Is it structured data with named fields?
|
||||
YES → @dataclass(frozen=True, slots=True)
|
||||
NO ↓
|
||||
Is it a tuple with positional semantics (x, y coords / DB row)?
|
||||
YES → NamedTuple
|
||||
NO → you probably don't need a new type
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Container reference
|
||||
|
||||
### @dataclass — internal value object
|
||||
|
||||
The default for structured data inside your codebase. Zero overhead, no framework coupling.
|
||||
|
||||
```python
|
||||
from dataclasses import dataclass
|
||||
from typing import NewType
|
||||
|
||||
UserId = NewType("UserId", int)
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class User:
|
||||
id: UserId
|
||||
name: str
|
||||
email: str
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Point:
|
||||
x: float
|
||||
y: float
|
||||
```
|
||||
|
||||
Always `frozen=True, slots=True`. Mutable only when mutation is the documented purpose — opt out with `# noqa: MUTABLE_OK`.
|
||||
|
||||
### Pydantic BaseModel — trust boundary guardian
|
||||
|
||||
Use when data enters or leaves your system. Validates at construction, serializes to JSON, generates OpenAPI schema.
|
||||
|
||||
```python
|
||||
from pydantic import BaseModel, ConfigDict, EmailStr
|
||||
|
||||
class CreateUserRequest(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
name: str
|
||||
email: EmailStr
|
||||
age: int
|
||||
|
||||
class UserResponse(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
id: int
|
||||
name: str
|
||||
email: str
|
||||
```
|
||||
|
||||
**The one rule**: data crosses a trust boundary → Pydantic. Everything else → dataclass.
|
||||
Never use Pydantic for internal-only data just because it's convenient. The validation cost is real.
|
||||
|
||||
### TypedDict — dict that knows its shape
|
||||
|
||||
Use when the value must stay a `dict` at runtime — JSON blobs, `**kwargs`, third-party APIs expecting dicts.
|
||||
|
||||
```python
|
||||
from typing import TypedDict, NotRequired
|
||||
|
||||
class Headers(TypedDict):
|
||||
content_type: str
|
||||
authorization: NotRequired[str]
|
||||
|
||||
def make_request(url: str, headers: Headers) -> None: ...
|
||||
|
||||
make_request("https://api.example.com", {"content_type": "application/json"})
|
||||
```
|
||||
|
||||
### Protocol — structural interface
|
||||
|
||||
"Anything that has method X" — no inheritance required.
|
||||
|
||||
```python
|
||||
from typing import Protocol
|
||||
|
||||
class Renderable(Protocol):
|
||||
def render(self) -> str: ...
|
||||
|
||||
class Saveable(Protocol):
|
||||
async def save(self) -> None: ...
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MarkdownDoc:
|
||||
content: str
|
||||
def render(self) -> str:
|
||||
return self.content
|
||||
|
||||
def publish(doc: Renderable) -> None:
|
||||
print(doc.render()) # MarkdownDoc works — no inheritance needed
|
||||
```
|
||||
|
||||
Default to Protocol for interfaces. ABC only when you need shared method implementations.
|
||||
|
||||
### ABC — interface with shared code
|
||||
|
||||
Only when Protocol isn't enough.
|
||||
|
||||
```python
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
class BaseRepository(ABC):
|
||||
@abstractmethod
|
||||
async def get(self, id: int) -> Model | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def save(self, model: Model) -> None: ...
|
||||
|
||||
async def get_or_raise(self, id: int) -> Model:
|
||||
result = await self.get(id)
|
||||
if result is None:
|
||||
msg = f"{type(self).__name__}: id {id} not found"
|
||||
raise LookupError(msg)
|
||||
return result
|
||||
```
|
||||
|
||||
### NamedTuple — positional + named (rare)
|
||||
|
||||
Only when you need tuple protocol (unpacking, indexing).
|
||||
|
||||
```python
|
||||
from typing import NamedTuple
|
||||
|
||||
class Coordinate(NamedTuple):
|
||||
x: float
|
||||
y: float
|
||||
|
||||
x, y = Coordinate(1.0, 2.0) # tuple unpacking
|
||||
```
|
||||
|
||||
99% of the time, `@dataclass(frozen=True, slots=True)` is better.
|
||||
|
||||
---
|
||||
|
||||
## Quick lookup
|
||||
|
||||
| Situation | Use | Why |
|
||||
|---|---|---|
|
||||
| User input, API request/response | `Pydantic BaseModel` | Validation, JSON schema, serialization |
|
||||
| DB row ↔ Python (ORM) | SQLAlchemy `Mapped[]` model | ORM integration, async session |
|
||||
| Internal value object | `@dataclass(frozen=True, slots=True)` | Zero overhead, no validation needed |
|
||||
| Multiple outcomes from function | Union of frozen dataclasses | Distinct types for `match` |
|
||||
| Dict shape for JSON / `**kwargs` | `TypedDict` | Stays a dict at runtime |
|
||||
| Fixed constants | `StrEnum` / `IntEnum` | Exhaustive match, no typos |
|
||||
| Distinct primitive | `NewType("X", int)` | Zero runtime cost, type-level only |
|
||||
| Contract / capability | `Protocol` | Structural typing, no inheritance |
|
||||
| Contract + shared impl | `ABC` | When Protocol isn't enough |
|
||||
|
||||
---
|
||||
|
||||
## Comparison matrix
|
||||
|
||||
| Feature | dataclass | Pydantic | TypedDict | Protocol | NamedTuple | NewType | Enum |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| Validation | - | ✓ | - | - | - | - | - |
|
||||
| JSON serialization | manual | built-in | native dict | - | - | - | `.value` |
|
||||
| Immutable | frozen=True | frozen=True | - (dict) | N/A | always | N/A | always |
|
||||
| Runtime cost | ~zero | validation | zero | zero | ~zero | zero | ~zero |
|
||||
| `match` support | ✓ | ✓ | - | - | ✓ | - | ✓ |
|
||||
| `slots` support | ✓ | - | - | - | - | - | - |
|
||||
|
||||
---
|
||||
|
||||
## Parse, don't validate
|
||||
|
||||
Validate at the boundary. Inside the boundary, types are proof of validity.
|
||||
|
||||
```python
|
||||
# BAD — validate then pass raw data
|
||||
def process_email(email: str) -> None:
|
||||
if "@" not in email:
|
||||
raise ValueError("invalid email")
|
||||
# still a raw str everywhere downstream
|
||||
|
||||
# GOOD — parse into typed value at boundary
|
||||
from typing import NewType
|
||||
|
||||
Email = NewType("Email", str)
|
||||
|
||||
def parse_email(raw: str) -> Email:
|
||||
if "@" not in raw or "." not in raw.split("@")[1]:
|
||||
msg = f"invalid email: {raw}"
|
||||
raise ValueError(msg)
|
||||
return Email(raw.lower().strip())
|
||||
|
||||
# Downstream only sees Email, never raw str
|
||||
def send_welcome(email: Email) -> None: ...
|
||||
```
|
||||
|
||||
With Pydantic this happens automatically — `EmailStr` is already a parsed type. Once constructed, `.email` is always valid. No re-validation needed.
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- Python docs: [dataclasses](https://docs.python.org/3/library/dataclasses.html)
|
||||
- Pydantic v2: [docs.pydantic.dev](https://docs.pydantic.dev/latest/)
|
||||
- Python docs: [typing — Protocol](https://docs.python.org/3/library/typing.html#typing.Protocol)
|
||||
- Python docs: [typing — TypedDict](https://docs.python.org/3/library/typing.html#typing.TypedDict)
|
||||
- Alexis King: [Parse, don't validate](https://lexi-lambda.github.io/blog/2019/11/05/parse-don-t-validate/)
|
||||
@@ -0,0 +1,133 @@
|
||||
# Data Processing — Polars + DuckDB
|
||||
|
||||
## The rule
|
||||
|
||||
NEVER pandas. Polars (with numpy) plus DuckDB. Pandas is 10-50x slower, has weaker types, and the modern Python data ecosystem has moved on.
|
||||
|
||||
## Quick decision tree
|
||||
|
||||
| Operation | Use | Why |
|
||||
|---|---|---|
|
||||
| `.csv` / `.parquet` / `.json` direct query | DuckDB | Zero memory load, SQL ergonomics |
|
||||
| `.duckdb` file | DuckDB | Native format |
|
||||
| Filter (any size) | Polars | 128x faster than DuckDB for filtering |
|
||||
| Sort | Polars | 12x faster |
|
||||
| Multi-table join | DuckDB | 3x faster, more join types |
|
||||
| Heavy GROUP BY aggregation | DuckDB | 4x faster on large datasets |
|
||||
| Window function | Polars | 3-5x faster |
|
||||
| Pivot / melt / string ops | Polars | 2x faster |
|
||||
| Larger than RAM | Polars streaming or DuckDB out-of-core | Both handle OOM |
|
||||
| Mixed pipeline | Hybrid (zero-copy via Arrow) | Use each tool's strengths |
|
||||
|
||||
For the deep version (per-operation benchmarks, OOM strategies, full execution templates), load the **`data-scientist`** skill - it lives in this same skill set and is the source of truth for performance numbers.
|
||||
|
||||
## Standard imports
|
||||
|
||||
```python
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
import duckdb
|
||||
```
|
||||
|
||||
## DuckDB direct file query (zero memory load)
|
||||
|
||||
```python
|
||||
result = duckdb.sql("""
|
||||
SELECT category, SUM(amount) AS total
|
||||
FROM 'data.csv'
|
||||
WHERE date >= '2026-01-01'
|
||||
GROUP BY category
|
||||
ORDER BY total DESC
|
||||
""").pl() # zero-copy → Polars DataFrame
|
||||
```
|
||||
|
||||
`.pl()` returns Polars; `.df()` would return pandas - never use `.df()`.
|
||||
|
||||
## Polars lazy pipeline
|
||||
|
||||
```python
|
||||
result = (
|
||||
pl.scan_csv("data.csv") # lazy, no read yet
|
||||
.filter(pl.col("amount") > 1000)
|
||||
.filter(pl.col("status") == "active")
|
||||
.sort("amount", descending=True)
|
||||
.head(100)
|
||||
.collect() # execute optimised plan
|
||||
)
|
||||
```
|
||||
|
||||
`scan_*` over `read_*` for files; `lazy()` then `collect()` for in-memory frames. Polars optimises the entire plan before execution (predicate pushdown, projection pushdown, common subexpression elimination).
|
||||
|
||||
## Streaming for OOM data
|
||||
|
||||
```python
|
||||
result = (
|
||||
pl.scan_csv("huge.csv")
|
||||
.filter(pl.col("active"))
|
||||
.group_by("category")
|
||||
.agg([
|
||||
pl.len().alias("count"),
|
||||
pl.sum("amount").alias("total"),
|
||||
])
|
||||
.collect(streaming=True)
|
||||
)
|
||||
```
|
||||
|
||||
## Hybrid pipeline (most realistic shape)
|
||||
|
||||
```python
|
||||
# Phase 1: DuckDB for the join (3x faster)
|
||||
joined = duckdb.sql("""
|
||||
SELECT o.*, c.region, p.category
|
||||
FROM 'orders.parquet' o
|
||||
JOIN 'customers.parquet' c ON o.customer_id = c.id
|
||||
JOIN 'products.parquet' p ON o.product_id = p.id
|
||||
""").pl()
|
||||
|
||||
# Phase 2: Polars for filtering and transformation (128x + 2x faster)
|
||||
processed = (
|
||||
joined
|
||||
.filter(pl.col("amount") > 100)
|
||||
.with_columns([
|
||||
(pl.col("amount") * 1.1).alias("amount_with_tax"),
|
||||
])
|
||||
)
|
||||
|
||||
# Phase 3: DuckDB for final aggregation (4x faster) - register Polars frame by name
|
||||
duckdb.register("processed", processed)
|
||||
final = duckdb.sql("""
|
||||
SELECT region, category, SUM(amount_with_tax) AS revenue
|
||||
FROM processed
|
||||
GROUP BY region, category
|
||||
ORDER BY revenue DESC
|
||||
""").pl()
|
||||
```
|
||||
|
||||
## Type safety with Polars
|
||||
|
||||
Polars supports schema overrides at read time, and `.cast()` for explicit conversion. Avoid implicit coercion in hot paths.
|
||||
|
||||
```python
|
||||
schema = {"id": pl.Int64, "amount": pl.Float64, "date": pl.Date}
|
||||
df = pl.read_csv("data.csv", schema_overrides=schema)
|
||||
```
|
||||
|
||||
basedpyright understands `polars-stubs`, which ship with polars itself. No extra type stubs to install.
|
||||
|
||||
## Things you might miss from pandas (and how to do them in Polars)
|
||||
|
||||
| pandas | polars |
|
||||
|---|---|
|
||||
| `df.iloc[5]` | `df.row(5)` (named tuple) or `df[5]` (single-row frame) |
|
||||
| `df.loc[df["x"] > 5]` | `df.filter(pl.col("x") > 5)` |
|
||||
| `df["x"].apply(fn)` | `df["x"].map_elements(fn)` (slow path) or use native expressions |
|
||||
| `df.merge(...)` | `df.join(other, on="key")` |
|
||||
| `df.groupby(...).agg(...)` | `df.group_by(...).agg(...)` |
|
||||
| `pd.read_csv(...).dtypes` | `pl.read_csv(...).schema` |
|
||||
| `df.to_dict("records")` | `df.to_dicts()` |
|
||||
|
||||
## Sources
|
||||
|
||||
- Polars docs: <https://docs.pola.rs>
|
||||
- DuckDB Python API: <https://duckdb.org/docs/api/python/overview>
|
||||
- Cross-reference - this skill set's `data-scientist` skill (load it for the deep version)
|
||||
@@ -0,0 +1,218 @@
|
||||
# Error Handling
|
||||
|
||||
Typed errors, exhaustive matching, union returns, and resource safety.
|
||||
|
||||
---
|
||||
|
||||
## Typed errors — no bare strings
|
||||
|
||||
Error types carry structured data. Pattern matching works. Callers know exactly what can go wrong.
|
||||
|
||||
```python
|
||||
from dataclasses import dataclass
|
||||
from typing import NewType
|
||||
|
||||
UserId = NewType("UserId", int)
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UserNotFoundError(Exception):
|
||||
user_id: UserId
|
||||
|
||||
def __str__(self) -> str: # REQUIRED — see note below
|
||||
return f"user {self.user_id} not found"
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PermissionDeniedError(Exception):
|
||||
user_id: UserId
|
||||
required_role: str
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"user {self.user_id} needs role {self.required_role}"
|
||||
```
|
||||
|
||||
**`__str__` is mandatory** on dataclass exceptions. `@dataclass` replaces `Exception.__init__`, so `self.args` is always `()`. Without `__str__`, `str(e)` returns an empty string and logging/monitoring breaks.
|
||||
|
||||
```python
|
||||
# BAD
|
||||
raise ValueError("user not found")
|
||||
raise ValueError("permission denied")
|
||||
|
||||
# GOOD
|
||||
raise UserNotFoundError(user_id=uid)
|
||||
raise PermissionDeniedError(user_id=uid, required_role="admin")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Union returns — expected failures without exceptions
|
||||
|
||||
For failures that are **expected** (not found, validation error, permission denied), return a union instead of raising. Exceptions are for **unexpected** failures (network down, OOM, corrupted data).
|
||||
|
||||
### Define the outcome types
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class User:
|
||||
id: UserId
|
||||
name: str
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UserNotFound:
|
||||
id: UserId
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PermissionDenied:
|
||||
id: UserId
|
||||
reason: str
|
||||
|
||||
type GetUserResult = User | UserNotFound | PermissionDenied
|
||||
```
|
||||
|
||||
### Handle exhaustively
|
||||
|
||||
```python
|
||||
from typing import assert_never
|
||||
|
||||
def handle_result(result: GetUserResult) -> str:
|
||||
match result:
|
||||
case User(name=name):
|
||||
return f"Found: {name}"
|
||||
case UserNotFound(id=uid):
|
||||
return f"No user with id {uid}"
|
||||
case PermissionDenied(reason=reason):
|
||||
return f"Denied: {reason}"
|
||||
case _ as unreachable:
|
||||
assert_never(unreachable)
|
||||
```
|
||||
|
||||
`assert_never` in the default case: if you add a new variant to `GetUserResult` without handling it here, the type checker errors. No silent fall-through.
|
||||
|
||||
### When to use which
|
||||
|
||||
**The heuristic**: caller is 1-2 levels away and MUST handle it → union return. Error should propagate up many layers to a boundary → exception.
|
||||
|
||||
| Scenario | Pattern | Why |
|
||||
|---|---|---|
|
||||
| Repository → service (caller handles it) | Union return (`User \| UserNotFound`) | Caller is right there, must handle both |
|
||||
| Validation at boundary (parsing input) | Exception (typed, with fields) | Propagates up to HTTP/CLI handler |
|
||||
| Infrastructure failure (network, OOM) | Exception | Can't handle locally, must propagate |
|
||||
| Service → service (deep internal) | Exception (typed) | Union boilerplate across many layers is worse than exceptions |
|
||||
| HTTP handler → response | Catch exceptions, convert to response | Boundary code catches and translates |
|
||||
|
||||
**Practical tradeoff**: union returns are safest (type checker forces handling) but create boilerplate when every caller in a chain must `match`. If the error would just propagate through 3+ layers unchanged, use a typed exception instead.
|
||||
|
||||
---
|
||||
|
||||
## Exhaustive match — every match needs a default
|
||||
|
||||
Every `match` statement ends with `case _: assert_never(x)`. No exceptions.
|
||||
|
||||
```python
|
||||
from enum import StrEnum
|
||||
from typing import assert_never
|
||||
|
||||
class Status(StrEnum):
|
||||
PENDING = "pending"
|
||||
ACTIVE = "active"
|
||||
DELETED = "deleted"
|
||||
|
||||
def describe(status: Status) -> str:
|
||||
match status:
|
||||
case Status.PENDING:
|
||||
return "waiting"
|
||||
case Status.ACTIVE:
|
||||
return "live"
|
||||
case Status.DELETED:
|
||||
return "gone"
|
||||
case _ as unreachable:
|
||||
assert_never(unreachable)
|
||||
```
|
||||
|
||||
Add a new enum member? The type checker tells you every `match` that needs updating.
|
||||
|
||||
---
|
||||
|
||||
## Context managers — resource safety
|
||||
|
||||
If it has `.close()`, `.shutdown()`, `.disconnect()`, or `.release()`, wrap it in `with`.
|
||||
|
||||
```python
|
||||
# BAD
|
||||
f = open("data.txt")
|
||||
data = f.read()
|
||||
f.close() # forgotten? leaked
|
||||
|
||||
# GOOD
|
||||
from pathlib import Path
|
||||
|
||||
data = Path("data.txt").read_text()
|
||||
```
|
||||
|
||||
### Async resources
|
||||
|
||||
```python
|
||||
import httpx
|
||||
|
||||
async def fetch_users() -> list[User]:
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get("https://api.example.com/users")
|
||||
response.raise_for_status()
|
||||
return [User(**u) for u in response.json()]
|
||||
```
|
||||
|
||||
### Custom context manager
|
||||
|
||||
```python
|
||||
from contextlib import asynccontextmanager
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
@asynccontextmanager
|
||||
async def managed_connection(url: str) -> AsyncIterator[Connection]:
|
||||
conn = await connect(url)
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
async with managed_connection("postgres://...") as conn:
|
||||
await conn.execute("SELECT 1")
|
||||
# conn is closed here, guaranteed
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Exception hierarchy — when you do raise
|
||||
|
||||
Keep exception hierarchies shallow and specific.
|
||||
|
||||
```python
|
||||
class AppError(Exception):
|
||||
"""Base for all application errors."""
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NotFoundError(AppError):
|
||||
entity: str
|
||||
id: int
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.entity} {self.id} not found"
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ConflictError(AppError):
|
||||
entity: str
|
||||
field: str
|
||||
value: str
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.entity}.{self.field} = {self.value!r} already exists"
|
||||
```
|
||||
|
||||
Callers catch `AppError` at the boundary, or specific subtypes where they can do something useful.
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- Python docs: [typing — assert_never](https://docs.python.org/3/library/typing.html#typing.assert_never)
|
||||
- Python docs: [contextlib](https://docs.python.org/3/library/contextlib.html)
|
||||
- Python docs: [match statement](https://docs.python.org/3/reference/compound_stmts.html#the-match-statement)
|
||||
@@ -0,0 +1,316 @@
|
||||
# FastAPI + SQLAlchemy 2.x async + Postgres + Pydantic v2
|
||||
|
||||
The canonical web API stack. Async end-to-end, type-safe end-to-end, OpenAPI-generated end-to-end.
|
||||
|
||||
## Project layout
|
||||
|
||||
```
|
||||
myapi/
|
||||
├── pyproject.toml
|
||||
├── alembic.ini
|
||||
├── migrations/
|
||||
│ └── env.py
|
||||
├── src/
|
||||
│ └── myapi/
|
||||
│ ├── __init__.py
|
||||
│ ├── main.py # FastAPI app + lifespan
|
||||
│ ├── config.py # pydantic-settings
|
||||
│ ├── db.py # engine, session factory, dependency
|
||||
│ ├── models.py # SQLAlchemy declarative models
|
||||
│ ├── schemas.py # Pydantic request/response models
|
||||
│ └── routers/
|
||||
│ ├── __init__.py
|
||||
│ └── users.py
|
||||
└── tests/
|
||||
├── conftest.py
|
||||
└── test_users.py
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
|
||||
```bash
|
||||
uv add fastapi 'sqlalchemy[asyncio]>=2.0' asyncpg 'pydantic[email]>=2' pydantic-settings 'uvicorn[standard]' orjson
|
||||
uv add --dev httpx pytest alembic
|
||||
```
|
||||
|
||||
`orjson` is mandatory: set `default_response_class=ORJSONResponse` on the FastAPI app. Pydantic-typed responses bypass it (Pydantic v2's `model_dump_json` is already Rust-backed); raw `dict` / `list` returns are accelerated. For SSE / NDJSON streams, call `orjson.dumps(...)` per chunk inside `StreamingResponse`. See `orjson-stack.md` for the decision tree, flag reference, and benchmarks.
|
||||
|
||||
## Configuration (`config.py`)
|
||||
|
||||
```python
|
||||
from functools import lru_cache
|
||||
|
||||
from pydantic import Field, PostgresDsn
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", env_prefix="MYAPI_")
|
||||
|
||||
database_url: PostgresDsn
|
||||
debug: bool = False
|
||||
cors_origins: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
return Settings() # type: ignore[call-arg] # pydantic populates from env
|
||||
```
|
||||
|
||||
Wait — that comment violates the no-excuse rule. Use proper field defaults instead. Real version:
|
||||
|
||||
```python
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", env_prefix="MYAPI_")
|
||||
database_url: PostgresDsn
|
||||
debug: bool = False
|
||||
cors_origins: list[str] = Field(default_factory=list)
|
||||
```
|
||||
|
||||
Construct via `Settings(_env_file=".env")` if needed in tests; in production it reads from env.
|
||||
|
||||
## Database (`db.py`)
|
||||
|
||||
```python
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncEngine,
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
|
||||
from myapi.config import get_settings
|
||||
|
||||
|
||||
def make_engine() -> AsyncEngine:
|
||||
settings = get_settings()
|
||||
return create_async_engine(
|
||||
str(settings.database_url),
|
||||
echo=settings.debug,
|
||||
pool_pre_ping=True,
|
||||
)
|
||||
|
||||
|
||||
_engine = make_engine()
|
||||
_SessionFactory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
|
||||
|
||||
async def get_session() -> AsyncIterator[AsyncSession]:
|
||||
async with _SessionFactory() as session:
|
||||
yield session
|
||||
|
||||
|
||||
SessionDep = Annotated[AsyncSession, Depends(get_session)]
|
||||
```
|
||||
|
||||
`expire_on_commit=False` is essential for FastAPI - otherwise attribute access after commit triggers an implicit refresh and errors out under async.
|
||||
|
||||
## Models (`models.py`)
|
||||
|
||||
```python
|
||||
from datetime import datetime, UTC
|
||||
from sqlalchemy import DateTime, String, func
|
||||
from sqlalchemy.orm import (
|
||||
DeclarativeBase,
|
||||
Mapped,
|
||||
MappedAsDataclass,
|
||||
mapped_column,
|
||||
)
|
||||
|
||||
|
||||
class Base(MappedAsDataclass, DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True, init=False)
|
||||
email: Mapped[str] = mapped_column(String(255), unique=True, index=True)
|
||||
name: Mapped[str] = mapped_column(String(100))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
server_default=func.now(),
|
||||
init=False,
|
||||
)
|
||||
```
|
||||
|
||||
`MappedAsDataclass` makes `User(email=..., name=...)` work as a real dataclass constructor. `init=False` excludes the auto-generated columns (`id`, `created_at`) from `__init__`.
|
||||
|
||||
## Schemas (`schemas.py`)
|
||||
|
||||
```python
|
||||
from datetime import datetime
|
||||
from pydantic import BaseModel, ConfigDict, EmailStr
|
||||
|
||||
|
||||
class UserCreate(BaseModel):
|
||||
email: EmailStr
|
||||
name: str
|
||||
|
||||
|
||||
class UserRead(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True) # SQLAlchemy → Pydantic
|
||||
|
||||
id: int
|
||||
email: EmailStr
|
||||
name: str
|
||||
created_at: datetime
|
||||
```
|
||||
|
||||
Always have a separate `*Create` (input) and `*Read` (output) model. Never expose your ORM model as the API model.
|
||||
|
||||
## Routers (`routers/users.py`)
|
||||
|
||||
```python
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
from sqlalchemy import select
|
||||
|
||||
from myapi.db import SessionDep
|
||||
from myapi.models import User
|
||||
from myapi.schemas import UserCreate, UserRead
|
||||
|
||||
router = APIRouter(prefix="/users", tags=["users"])
|
||||
|
||||
|
||||
@router.post("", response_model=UserRead, status_code=status.HTTP_201_CREATED)
|
||||
async def create_user(payload: UserCreate, session: SessionDep) -> User:
|
||||
user = User(email=payload.email, name=payload.name)
|
||||
session.add(user)
|
||||
await session.commit()
|
||||
await session.refresh(user)
|
||||
return user
|
||||
|
||||
|
||||
@router.get("/{user_id}", response_model=UserRead)
|
||||
async def get_user(user_id: int, session: SessionDep) -> User:
|
||||
result = await session.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if user is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "User not found")
|
||||
return user
|
||||
|
||||
|
||||
@router.get("", response_model=list[UserRead])
|
||||
async def list_users(session: SessionDep, limit: int = 100) -> list[User]:
|
||||
result = await session.execute(select(User).limit(limit))
|
||||
return list(result.scalars().all())
|
||||
```
|
||||
|
||||
## Application (`main.py`)
|
||||
|
||||
```python
|
||||
from contextlib import asynccontextmanager
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from myapi.config import get_settings
|
||||
from myapi.routers import users
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
# Startup: warm up engine pool, run migrations check, etc.
|
||||
yield
|
||||
# Shutdown: close engine
|
||||
from myapi.db import _engine
|
||||
await _engine.dispose()
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
settings = get_settings()
|
||||
app = FastAPI(
|
||||
title="My API",
|
||||
debug=settings.debug,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
app.include_router(users.router)
|
||||
return app
|
||||
|
||||
|
||||
app = create_app()
|
||||
```
|
||||
|
||||
Run with:
|
||||
|
||||
```bash
|
||||
uv run uvicorn myapi.main:app --host 0.0.0.0 --port 8000 --reload
|
||||
```
|
||||
|
||||
## Migrations (Alembic + async)
|
||||
|
||||
```bash
|
||||
uv run alembic init -t async migrations
|
||||
```
|
||||
|
||||
In `migrations/env.py` replace the `target_metadata` line:
|
||||
|
||||
```python
|
||||
from myapi.models import Base
|
||||
target_metadata = Base.metadata
|
||||
```
|
||||
|
||||
Set `sqlalchemy.url` in `alembic.ini` to your async URL or override via `env.py`:
|
||||
|
||||
```python
|
||||
from myapi.config import get_settings
|
||||
config.set_main_option("sqlalchemy.url", str(get_settings().database_url))
|
||||
```
|
||||
|
||||
Generate and apply:
|
||||
|
||||
```bash
|
||||
uv run alembic revision --autogenerate -m "create users"
|
||||
uv run alembic upgrade head
|
||||
```
|
||||
|
||||
## Tests (`tests/test_users.py`)
|
||||
|
||||
```python
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from myapi.main import app
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_create_and_get_user() -> None:
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||
create_response = await client.post(
|
||||
"/users",
|
||||
json={"email": "alice@example.com", "name": "Alice"},
|
||||
)
|
||||
assert create_response.status_code == 201
|
||||
user_id = create_response.json()["id"]
|
||||
|
||||
get_response = await client.get(f"/users/{user_id}")
|
||||
assert get_response.status_code == 200
|
||||
assert get_response.json()["email"] == "alice@example.com"
|
||||
```
|
||||
|
||||
For database-backed tests, run a Postgres container in CI (`testcontainers-python` or `docker-compose`) and apply migrations against a test schema. SQLite-as-test-db breaks once you use Postgres-specific types (`JSONB`, `tsvector`, arrays).
|
||||
|
||||
## Common pitfalls
|
||||
|
||||
| Pitfall | Fix |
|
||||
|---|---|
|
||||
| `MissingGreenlet` exception when accessing relationships after commit | `expire_on_commit=False` on the session factory |
|
||||
| Connection pool exhausted under load | Set `pool_size`, `max_overflow` in `create_async_engine` |
|
||||
| Pydantic v1 syntax (`from pydantic import ...; class X(BaseModel): class Config: orm_mode = True`) | v2 uses `model_config = ConfigDict(from_attributes=True)` |
|
||||
| Returning ORM objects without `response_model` | FastAPI serialises with `from_attributes=True` automatically; declare `response_model` so OpenAPI is correct |
|
||||
| `await session.execute(...)` returning Sequence | Wrap with `list(result.scalars().all())` to satisfy strict types |
|
||||
| `func.now()` returning naive datetime | Use `DateTime(timezone=True)` and `created_at: Mapped[datetime]` with `UTC`-aware default |
|
||||
|
||||
## Sources
|
||||
|
||||
- FastAPI: <https://fastapi.tiangolo.com>
|
||||
- SQLAlchemy 2.x async: <https://docs.sqlalchemy.org/en/20/orm/extensions/asyncio.html>
|
||||
- SQLAlchemy MappedAsDataclass: <https://docs.sqlalchemy.org/en/20/orm/dataclasses.html>
|
||||
- asyncpg: <https://magicstack.github.io/asyncpg/current/>
|
||||
- Pydantic v2 migration: <https://docs.pydantic.dev/latest/migration/>
|
||||
- Alembic async: <https://alembic.sqlalchemy.org/en/latest/cookbook.html#using-asyncio-with-alembic>
|
||||
+360
@@ -0,0 +1,360 @@
|
||||
# httpx2 — Production Defaults
|
||||
|
||||
> **Source**: [pydantic/httpx2](https://github.com/pydantic/httpx2) — next-generation HTTP client for Python 3, continuation of HTTPX under Pydantic stewardship.
|
||||
>
|
||||
> **Rule**: Every network request MUST use `httpx2`. **ALL optimizations below are ON by default** — HTTP/2, brotli+zstd, tuned connection pool, fine-grained timeouts, transport retries, TCP_NODELAY. This is the baseline, not a stretch goal. A bare `httpx2.AsyncClient()` is a bug.
|
||||
|
||||
---
|
||||
|
||||
## 1. Installation — all extras, always
|
||||
|
||||
```toml
|
||||
# pyproject.toml
|
||||
dependencies = [
|
||||
"httpx2[http2,brotli,zstd]",
|
||||
]
|
||||
```
|
||||
|
||||
| Extra | What it enables | Why it's mandatory |
|
||||
|-------|----------------|--------------------|
|
||||
| `http2` | HTTP/2 multiplexing via `h2` | Single TCP connection handles concurrent requests; eliminates head-of-line blocking |
|
||||
| `brotli` | Brotli content decoding (`br`) | ~20% smaller payloads than gzip for text/JSON |
|
||||
| `zstd` | Zstandard content decoding | Faster decompression than brotli at similar ratios; stdlib in Python ≥ 3.14 |
|
||||
| `socks` | SOCKS5 proxy support via `socksio` | Install only if you route through SOCKS proxies |
|
||||
|
||||
All three core extras (`http2,brotli,zstd`) are non-negotiable. Omitting any is leaving performance on the table.
|
||||
|
||||
---
|
||||
|
||||
## 2. The canonical defaults — ALL ON
|
||||
|
||||
These are not "optimizations to consider". These are **the correct defaults** that every httpx2 client must use.
|
||||
|
||||
```python
|
||||
import socket
|
||||
import httpx2
|
||||
|
||||
# ── These are the STANDARD values. Use them verbatim. ──
|
||||
|
||||
LIMITS = httpx2.Limits(
|
||||
max_connections=200, # library default 100 is too conservative
|
||||
max_keepalive_connections=40, # library default 20 wastes reconnects
|
||||
keepalive_expiry=30.0, # library default 5s kills warm connections too fast
|
||||
)
|
||||
|
||||
TIMEOUT = httpx2.Timeout(
|
||||
connect=5.0, # TCP + TLS handshake budget
|
||||
read=30.0, # time to receive a response chunk
|
||||
write=10.0, # time to send a request chunk
|
||||
pool=10.0, # time to acquire a connection from pool
|
||||
)
|
||||
|
||||
SOCKET_OPTIONS: list[tuple[int, int, int]] = [
|
||||
(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1), # disable Nagle — no 40ms delay
|
||||
]
|
||||
```
|
||||
|
||||
### Why each knob is set this way
|
||||
|
||||
| Setting | Library default | Our default | Why |
|
||||
|---------|----------------|-------------|-----|
|
||||
| `http2` | `False` | **`True`** | HTTP/2 multiplexing is strictly superior for any modern API |
|
||||
| `max_connections` | `100` | `200` | Headroom for fan-out; prevents pool exhaustion under load |
|
||||
| `max_keepalive_connections` | `20` | `40` | Keeps warm connections alive; fewer TLS handshakes |
|
||||
| `keepalive_expiry` | `5.0s` | `30.0s` | 5s is too aggressive — kills connections between burst requests |
|
||||
| `Timeout(5.0)` uniform | `5.0` all | Split | Uniform 5s is too tight for reads, too loose for connects |
|
||||
| `read` timeout | `5.0` | `30.0` | Slow APIs and streaming need breathing room |
|
||||
| `pool` timeout | `5.0` | `10.0` | Explicit — hitting this means `max_connections` needs raising |
|
||||
| `TCP_NODELAY` | off | **on** | Eliminates Nagle's 40ms coalescing delay for small payloads |
|
||||
| `retries` | `0` | `3` | Retries on `ConnectError`/`ConnectTimeout` only — safe and resilient |
|
||||
| `follow_redirects` | `False` | **`True`** | Most APIs redirect; failing on 3xx is wrong default behavior |
|
||||
|
||||
---
|
||||
|
||||
## 3. Factory functions — the ONE correct way to create clients
|
||||
|
||||
Copy this into your project. This is the canonical pattern.
|
||||
|
||||
```python
|
||||
"""httpx2 client factory. Always use create_client() / create_async_client()."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import typing
|
||||
|
||||
import httpx2
|
||||
|
||||
_LIMITS = httpx2.Limits(
|
||||
max_connections=200,
|
||||
max_keepalive_connections=40,
|
||||
keepalive_expiry=30.0,
|
||||
)
|
||||
|
||||
_TIMEOUT = httpx2.Timeout(
|
||||
connect=5.0,
|
||||
read=30.0,
|
||||
write=10.0,
|
||||
pool=10.0,
|
||||
)
|
||||
|
||||
_SOCKET_OPTIONS: list[tuple[int, int, int]] = [
|
||||
(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1),
|
||||
]
|
||||
|
||||
|
||||
def create_async_client(
|
||||
*,
|
||||
base_url: str = "",
|
||||
http2: bool = True,
|
||||
retries: int = 3,
|
||||
limits: httpx2.Limits = _LIMITS,
|
||||
timeout: httpx2.Timeout = _TIMEOUT,
|
||||
headers: dict[str, str] | None = None,
|
||||
event_hooks: dict[str, list[typing.Callable[..., typing.Any]]] | None = None,
|
||||
**kwargs: typing.Any,
|
||||
) -> httpx2.AsyncClient:
|
||||
transport = httpx2.AsyncHTTPTransport(
|
||||
http2=http2,
|
||||
retries=retries,
|
||||
limits=limits,
|
||||
socket_options=_SOCKET_OPTIONS,
|
||||
)
|
||||
return httpx2.AsyncClient(
|
||||
transport=transport,
|
||||
timeout=timeout,
|
||||
base_url=base_url,
|
||||
headers=headers or {},
|
||||
event_hooks=event_hooks or {},
|
||||
follow_redirects=True,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def create_client(
|
||||
*,
|
||||
base_url: str = "",
|
||||
http2: bool = True,
|
||||
retries: int = 3,
|
||||
limits: httpx2.Limits = _LIMITS,
|
||||
timeout: httpx2.Timeout = _TIMEOUT,
|
||||
headers: dict[str, str] | None = None,
|
||||
event_hooks: dict[str, list[typing.Callable[..., typing.Any]]] | None = None,
|
||||
**kwargs: typing.Any,
|
||||
) -> httpx2.Client:
|
||||
transport = httpx2.HTTPTransport(
|
||||
http2=http2,
|
||||
retries=retries,
|
||||
limits=limits,
|
||||
socket_options=_SOCKET_OPTIONS,
|
||||
)
|
||||
return httpx2.Client(
|
||||
transport=transport,
|
||||
timeout=timeout,
|
||||
base_url=base_url,
|
||||
headers=headers or {},
|
||||
event_hooks=event_hooks or {},
|
||||
follow_redirects=True,
|
||||
**kwargs,
|
||||
)
|
||||
```
|
||||
|
||||
Usage:
|
||||
|
||||
```python
|
||||
# Async — the common case
|
||||
async with create_async_client(base_url="https://api.example.com") as client:
|
||||
r = await client.get("/users")
|
||||
|
||||
# Sync
|
||||
with create_client() as client:
|
||||
r = client.get("https://api.example.com/health")
|
||||
```
|
||||
|
||||
**If you are NOT using this factory pattern, you are doing it wrong.** A bare `httpx2.AsyncClient()` leaves HTTP/2 off, retries off, TCP_NODELAY off, keepalive too short, and timeouts too uniform.
|
||||
|
||||
---
|
||||
|
||||
## 4. Special case overrides
|
||||
|
||||
The factory defaults cover 95% of use cases. Override only when you have a specific reason:
|
||||
|
||||
| Scenario | Override |
|
||||
|----------|----------|
|
||||
| LLM streaming endpoints | `timeout=httpx2.Timeout(connect=10.0, read=None, write=10.0, pool=10.0)` — no read timeout on streaming |
|
||||
| Single-host API with low concurrency | `limits=httpx2.Limits(max_connections=50, max_keepalive_connections=20, keepalive_expiry=60.0)` |
|
||||
| Ephemeral short-lived requests | `keepalive_expiry=5.0` — don't hold connections |
|
||||
| Unix domain sockets | `httpx2.AsyncHTTPTransport(uds="/path/to/socket", ...)` |
|
||||
| mTLS / client certs | Pass `verify=ssl_ctx` with `ctx.load_cert_chain(certfile=...)` |
|
||||
| SOCKS proxy | `httpx2[socks]`, `proxy="socks5://..."` |
|
||||
|
||||
---
|
||||
|
||||
## 5. Event hooks — always wire observability
|
||||
|
||||
This is not optional. Every production client should log requests.
|
||||
|
||||
```python
|
||||
import time
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
async def log_request(request: httpx2.Request) -> None:
|
||||
request.extensions["request_start"] = time.perf_counter()
|
||||
|
||||
async def log_response(response: httpx2.Response) -> None:
|
||||
start = response.request.extensions.get("request_start", 0)
|
||||
elapsed = time.perf_counter() - start
|
||||
logger.info(
|
||||
"HTTP %s %s → %d (%.3fs, %s)",
|
||||
response.request.method,
|
||||
response.request.url,
|
||||
response.status_code,
|
||||
elapsed,
|
||||
response.http_version,
|
||||
)
|
||||
|
||||
# Sync versions for Client
|
||||
def log_request_sync(request: httpx2.Request) -> None:
|
||||
request.extensions["request_start"] = time.perf_counter()
|
||||
|
||||
def log_response_sync(response: httpx2.Response) -> None:
|
||||
start = response.request.extensions.get("request_start", 0)
|
||||
elapsed = time.perf_counter() - start
|
||||
logger.info(
|
||||
"HTTP %s %s → %d (%.3fs, %s)",
|
||||
response.request.method,
|
||||
response.request.url,
|
||||
response.status_code,
|
||||
elapsed,
|
||||
response.http_version,
|
||||
)
|
||||
```
|
||||
|
||||
For auto `raise_for_status()`:
|
||||
|
||||
```python
|
||||
async def raise_on_error(response: httpx2.Response) -> None:
|
||||
response.raise_for_status()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Verification script — confirm your setup is fully optimized
|
||||
|
||||
Run this against your target endpoint to **verify** (not decide) that all optimizations are active:
|
||||
|
||||
```python
|
||||
"""Verify httpx2 is fully optimized against a target endpoint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import time
|
||||
|
||||
import anyio
|
||||
import httpx2
|
||||
|
||||
|
||||
TARGET_URL = "https://api.example.com/health"
|
||||
ITERATIONS = 30
|
||||
|
||||
|
||||
async def bench(label: str, client: httpx2.AsyncClient, url: str, n: int) -> float:
|
||||
for _ in range(3): # warmup
|
||||
await client.get(url)
|
||||
start = time.perf_counter()
|
||||
for _ in range(n):
|
||||
r = await client.get(url)
|
||||
assert r.status_code == 200
|
||||
elapsed = time.perf_counter() - start
|
||||
avg_ms = (elapsed / n) * 1000
|
||||
print(f" {label}: {avg_ms:.1f}ms avg ({n} reqs in {elapsed:.2f}s)")
|
||||
return avg_ms
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
results: dict[str, float] = {}
|
||||
|
||||
# BAD: bare defaults (this is what we're proving is worse)
|
||||
async with httpx2.AsyncClient() as c:
|
||||
results["BAD-bare-defaults"] = await bench("BAD-bare-defaults", c, TARGET_URL, ITERATIONS)
|
||||
|
||||
# GOOD: full production defaults (this is what we always use)
|
||||
limits = httpx2.Limits(max_connections=200, max_keepalive_connections=40, keepalive_expiry=30.0)
|
||||
timeout = httpx2.Timeout(connect=5.0, read=30.0, write=10.0, pool=10.0)
|
||||
transport = httpx2.AsyncHTTPTransport(
|
||||
http2=True, retries=3, limits=limits,
|
||||
socket_options=[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)],
|
||||
)
|
||||
async with httpx2.AsyncClient(transport=transport, timeout=timeout, follow_redirects=True) as c:
|
||||
results["GOOD-full-production"] = await bench("GOOD-full-production", c, TARGET_URL, ITERATIONS)
|
||||
|
||||
print("\n--- Proof ---")
|
||||
baseline = results["BAD-bare-defaults"]
|
||||
for label, avg in results.items():
|
||||
delta = ((avg - baseline) / baseline) * 100
|
||||
print(f" {label}: {avg:.1f}ms ({delta:+.1f}% vs bare)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
anyio.run(main)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. Quick reference — all knobs
|
||||
|
||||
### `httpx2.AsyncClient` / `httpx2.Client`
|
||||
|
||||
| Parameter | Type | Library Default | **Our Default** |
|
||||
|-----------|------|-----------------|-----------------|
|
||||
| `http1` | `bool` | `True` | `True` |
|
||||
| `http2` | `bool` | `False` | **`True`** |
|
||||
| `verify` | `ssl.SSLContext \| str \| bool` | `True` | `True` |
|
||||
| `cert` | `CertTypes \| None` | `None` | `None` |
|
||||
| `proxy` | `str \| Proxy \| None` | `None` | `None` |
|
||||
| `mounts` | `dict[str, Transport]` | `None` | `None` |
|
||||
| `timeout` | `Timeout \| float \| None` | `Timeout(5.0)` | **Split: 5/30/10/10** |
|
||||
| `limits` | `Limits` | `Limits(100, 20, 5.0)` | **`Limits(200, 40, 30.0)`** |
|
||||
| `follow_redirects` | `bool` | `False` | **`True`** |
|
||||
| `max_redirects` | `int` | `20` | `20` |
|
||||
| `event_hooks` | `dict` | `{}` | **Wire logging** |
|
||||
| `base_url` | `str` | `""` | Set for single-API clients |
|
||||
| `trust_env` | `bool` | `True` | `True` |
|
||||
| `default_encoding` | `str \| Callable` | `"utf-8"` | `"utf-8"` |
|
||||
|
||||
### `httpx2.AsyncHTTPTransport` / `httpx2.HTTPTransport`
|
||||
|
||||
| Parameter | Type | Library Default | **Our Default** |
|
||||
|-----------|------|-----------------|-----------------|
|
||||
| `http1` | `bool` | `True` | `True` |
|
||||
| `http2` | `bool` | `False` | **`True`** |
|
||||
| `retries` | `int` | `0` | **`3`** |
|
||||
| `limits` | `Limits` | `Limits(100, 20, 5.0)` | **`Limits(200, 40, 30.0)`** |
|
||||
| `uds` | `str \| None` | `None` | `None` |
|
||||
| `local_address` | `str \| None` | `None` | `None` |
|
||||
| `socket_options` | `Iterable[SOCKET_OPTION]` | `None` | **`[TCP_NODELAY]`** |
|
||||
| `proxy` | `str \| Proxy \| None` | `None` | `None` |
|
||||
|
||||
### `httpx2.Timeout`
|
||||
|
||||
| Parameter | Library Default | **Our Default** |
|
||||
|-----------|-----------------|-----------------|
|
||||
| `connect` | `5.0` | `5.0` |
|
||||
| `read` | `5.0` | **`30.0`** |
|
||||
| `write` | `5.0` | **`10.0`** |
|
||||
| `pool` | `5.0` | **`10.0`** |
|
||||
|
||||
### `httpx2.Limits`
|
||||
|
||||
| Parameter | Library Default | **Our Default** |
|
||||
|-----------|-----------------|-----------------|
|
||||
| `max_connections` | `100` | **`200`** |
|
||||
| `max_keepalive_connections` | `20` | **`40`** |
|
||||
| `keepalive_expiry` | `5.0` | **`30.0`** |
|
||||
|
||||
### Async backend (httpcore2)
|
||||
|
||||
httpcore2 uses `anyio` by default (works with both asyncio and trio). No extra config needed if you're already on the anyio stack. For trio, install `httpcore2[trio]`.
|
||||
@@ -0,0 +1,307 @@
|
||||
# Library Defaults — Decision Tree
|
||||
|
||||
For each domain, the canonical 2026 choice, why, and the canonical usage snippet. The skill enforces these unless the project's `pyproject.toml` explicitly says otherwise.
|
||||
|
||||
## CLI — typer
|
||||
|
||||
`typer` builds a CLI from type-annotated function signatures. argparse needs 5x the code; click ignores type annotations; fire is magic that breaks at scale.
|
||||
|
||||
```python
|
||||
import typer
|
||||
from rich import print as rprint
|
||||
|
||||
app = typer.Typer()
|
||||
|
||||
@app.command()
|
||||
def greet(name: str, count: int = 1, shout: bool = False) -> None:
|
||||
"""Print a greeting `count` times."""
|
||||
message = f"Hello, {name}!" if not shout else f"HELLO, {name.upper()}!"
|
||||
for _ in range(count):
|
||||
rprint(message)
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
```
|
||||
|
||||
For a single-function script, `typer.run(main)` skips the `Typer()` boilerplate. Subcommands use `@app.command()`.
|
||||
|
||||
## Terminal output — rich
|
||||
|
||||
`rich` produces tables, progress bars, syntax highlighting, traceback rendering. Use it for any structured output. Plain `print` is acceptable for non-interactive log lines (and even those are usually better via `rich.console.Console(stderr=True).log(...)`).
|
||||
|
||||
```python
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Users")
|
||||
table.add_column("ID", style="cyan")
|
||||
table.add_column("Name", style="magenta")
|
||||
table.add_row("1", "Alice")
|
||||
console.print(table)
|
||||
|
||||
# Rich tracebacks (call once at process start)
|
||||
from rich.traceback import install
|
||||
install(show_locals=True)
|
||||
```
|
||||
|
||||
## HTTP client — [httpx2](https://github.com/pydantic/httpx2)
|
||||
|
||||
Next-generation HTTP client under Pydantic stewardship. Sync and async in one library, HTTP/2 native, brotli + zstd content decoding, real type stubs. Replaces `requests` (sync only), `aiohttp` (async only), and the original `httpx`.
|
||||
|
||||
**Install**: `httpx2[http2,brotli,zstd]` — always include all three extras, no exceptions.
|
||||
|
||||
**A bare `httpx2.AsyncClient()` / `httpx2.Client()` is a bug.** Always use the factory pattern from `references/httpx2-optimization.md` with ALL optimizations enabled by default:
|
||||
|
||||
```python
|
||||
import socket
|
||||
import httpx2
|
||||
|
||||
# ── Production defaults — ALL ON, always. ──
|
||||
_LIMITS = httpx2.Limits(max_connections=200, max_keepalive_connections=40, keepalive_expiry=30.0)
|
||||
_TIMEOUT = httpx2.Timeout(connect=5.0, read=30.0, write=10.0, pool=10.0)
|
||||
_SOCKET_OPTS: list[tuple[int, int, int]] = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
|
||||
|
||||
# Async (the common case)
|
||||
transport = httpx2.AsyncHTTPTransport(http2=True, retries=3, limits=_LIMITS, socket_options=_SOCKET_OPTS)
|
||||
async with httpx2.AsyncClient(transport=transport, timeout=_TIMEOUT, follow_redirects=True) as client:
|
||||
response = await client.get("https://api.example.com/users")
|
||||
response.raise_for_status()
|
||||
users = response.json()
|
||||
|
||||
# Sync
|
||||
transport = httpx2.HTTPTransport(http2=True, retries=3, limits=_LIMITS, socket_options=_SOCKET_OPTS)
|
||||
with httpx2.Client(transport=transport, timeout=_TIMEOUT, follow_redirects=True) as client:
|
||||
response = client.get("https://api.example.com/users")
|
||||
response.raise_for_status()
|
||||
users = response.json()
|
||||
```
|
||||
|
||||
See `references/httpx2-optimization.md` for the full factory functions (`create_client()` / `create_async_client()`), event hooks, and the rationale behind every setting. **Load that reference whenever you write ANY network code.**
|
||||
|
||||
## JSON — stdlib `json` (default) or `orjson` (hot paths)
|
||||
|
||||
Stdlib `json` is fine for cold paths and configs. **Reach for `orjson` when JSON is in the hot path** — cache layers, queue payloads, streaming responses, structured logs, FastAPI endpoints returning raw `dict` / `list`.
|
||||
|
||||
```python
|
||||
import orjson
|
||||
|
||||
# orjson.dumps returns bytes, not str
|
||||
raw: bytes = orjson.dumps(
|
||||
payload,
|
||||
option=orjson.OPT_NAIVE_UTC | orjson.OPT_UTC_Z | orjson.OPT_SERIALIZE_DATACLASS,
|
||||
)
|
||||
```
|
||||
|
||||
**Critical 2026 fact**: with Pydantic v2, `model.model_dump_json()` is backed by pydantic-core (Rust) and is faster than `orjson + default=` bridge for Pydantic-shaped responses. **Use `model_dump_json()` for Pydantic; orjson for everything else.**
|
||||
|
||||
For FastAPI: `app = FastAPI(default_response_class=ORJSONResponse)`. Pydantic-typed responses bypass it (and that's correct — Pydantic's path is faster). Raw `dict`/`list` returns go through orjson.
|
||||
|
||||
See `references/orjson-stack.md` for the full decision tree, option flag reference, FastAPI integration, Redis/queue/logging patterns, and the `model_dump_json()` vs orjson benchmark.
|
||||
|
||||
## Validation — pydantic v2
|
||||
|
||||
Pydantic v2's core is in Rust (~10x faster than v1). It is the de-facto boundary validator. Use it for:
|
||||
|
||||
- HTTP request/response models (FastAPI uses pydantic natively)
|
||||
- Config files (env vars via `pydantic-settings`)
|
||||
- Anything entering the program from outside
|
||||
|
||||
```python
|
||||
from pydantic import BaseModel, Field, EmailStr, field_validator
|
||||
|
||||
class User(BaseModel):
|
||||
id: int = Field(ge=1)
|
||||
email: EmailStr
|
||||
name: str = Field(min_length=1, max_length=100)
|
||||
age: int | None = Field(default=None, ge=0, le=150)
|
||||
|
||||
@field_validator("name")
|
||||
@classmethod
|
||||
def name_no_digits(cls, v: str) -> str:
|
||||
if any(c.isdigit() for c in v):
|
||||
raise ValueError("name cannot contain digits")
|
||||
return v
|
||||
|
||||
# Inside the program, use the validated instance with confidence
|
||||
user = User.model_validate({"id": 1, "email": "a@b.com", "name": "Alice"})
|
||||
print(user.model_dump_json(indent=2))
|
||||
```
|
||||
|
||||
`@dataclass` is fine for purely internal records (no validation needed). For anything crossing a process boundary, use Pydantic.
|
||||
|
||||
## Async — anyio
|
||||
|
||||
Full reference: [async-anyio.md](async-anyio.md). The summary:
|
||||
|
||||
```python
|
||||
import anyio
|
||||
|
||||
async def fetch(url: str) -> str:
|
||||
await anyio.sleep(0.1)
|
||||
return url
|
||||
|
||||
async def main() -> None:
|
||||
async with anyio.create_task_group() as tg:
|
||||
for url in ["a", "b", "c"]:
|
||||
tg.start_soon(fetch, url)
|
||||
|
||||
anyio.run(main)
|
||||
```
|
||||
|
||||
Never `import asyncio` directly. The third-party libraries you call are free to use asyncio internally.
|
||||
|
||||
## Web framework — fastapi
|
||||
|
||||
Type-hint-driven HTTP framework. Pydantic models become OpenAPI schemas automatically.
|
||||
|
||||
```python
|
||||
from fastapi import FastAPI
|
||||
from pydantic import BaseModel
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
class CreateUser(BaseModel):
|
||||
name: str
|
||||
email: str
|
||||
|
||||
class User(BaseModel):
|
||||
id: int
|
||||
name: str
|
||||
email: str
|
||||
|
||||
@app.post("/users", response_model=User)
|
||||
async def create_user(payload: CreateUser) -> User:
|
||||
return User(id=1, **payload.model_dump())
|
||||
```
|
||||
|
||||
Full stack with database: [fastapi-stack.md](fastapi-stack.md).
|
||||
|
||||
## ORM — sqlalchemy 2.x async
|
||||
|
||||
SQLAlchemy 2.x finally has a real async API. Use the modern declarative `MappedAsDataclass` style with type annotations.
|
||||
|
||||
```python
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, MappedAsDataclass
|
||||
|
||||
class Base(MappedAsDataclass, DeclarativeBase):
|
||||
pass
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
id: Mapped[int] = mapped_column(primary_key=True, init=False)
|
||||
name: Mapped[str] = mapped_column(String(100))
|
||||
email: Mapped[str] = mapped_column(String(255), unique=True)
|
||||
|
||||
engine = create_async_engine("postgresql+asyncpg://localhost/myapp")
|
||||
SessionFactory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
```
|
||||
|
||||
Full pattern with FastAPI integration: [fastapi-stack.md](fastapi-stack.md).
|
||||
|
||||
## Database — postgres + asyncpg
|
||||
|
||||
For new applications, default to Postgres. SQLite for tests is fine; SQLite for production is not.
|
||||
|
||||
asyncpg is the fastest Python Postgres driver, native to SQLAlchemy 2.x async, native to FastAPI's lifespan model. URL: `postgresql+asyncpg://user:pass@host:5432/db`.
|
||||
|
||||
For migrations, use Alembic with `[alembic.context]` configured to use the async engine. Single-step:
|
||||
|
||||
```bash
|
||||
uv add alembic
|
||||
uv run alembic init -t async migrations
|
||||
```
|
||||
|
||||
## TUI — textual
|
||||
|
||||
Textual builds rich, mouse-aware, mobile-style TUIs on the rich rendering engine. See [textual-tui.md](textual-tui.md).
|
||||
|
||||
## AI agents — pydantic-ai
|
||||
|
||||
The agent framework from the Pydantic team. Type-strict, structured outputs are first-class, model-agnostic. See [pydantic-ai.md](pydantic-ai.md).
|
||||
|
||||
## DataFrames — polars + numpy
|
||||
|
||||
Polars is 10-50x faster than pandas, has a real type system, and supports lazy evaluation. Numpy stays in the toolbox for arrays. See [data-processing.md](data-processing.md).
|
||||
|
||||
## OLAP / SQL — duckdb
|
||||
|
||||
DuckDB is the SQL engine for analytical workloads. Query CSV/Parquet/JSON files directly without loading into memory; perform joins and aggregations 3-4x faster than Polars; zero-copy interchange with Polars via Arrow. See [data-processing.md](data-processing.md).
|
||||
|
||||
## Tests — pytest
|
||||
|
||||
Plain `unittest` is fine for stdlib; everything else uses pytest. Conventions:
|
||||
|
||||
- File names `test_*.py`, function names `test_*`.
|
||||
- Fixtures via `@pytest.fixture`. Async fixtures are anyio-aware (`@pytest.fixture` on an async function works under `pytest-anyio` which is bundled with anyio).
|
||||
- Parametrise with `@pytest.mark.parametrize`.
|
||||
- Mark async tests with `@pytest.mark.anyio` (provided by anyio's pytest plugin).
|
||||
|
||||
```python
|
||||
import pytest
|
||||
import anyio
|
||||
|
||||
@pytest.fixture
|
||||
def sample_user() -> dict[str, str]:
|
||||
return {"name": "Alice", "email": "a@b.com"}
|
||||
|
||||
@pytest.mark.parametrize("count,expected", [(1, "Hello"), (2, "Hello, Hello")])
|
||||
def test_greet(count: int, expected: str) -> None:
|
||||
result = ", ".join(["Hello"] * count)
|
||||
assert result == expected
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_async_fetch() -> None:
|
||||
await anyio.sleep(0)
|
||||
assert True
|
||||
```
|
||||
|
||||
`pyproject.toml`:
|
||||
|
||||
```toml
|
||||
[tool.pytest.ini_options]
|
||||
minversion = "8.0"
|
||||
testpaths = ["tests"]
|
||||
addopts = ["-ra", "--strict-config", "--strict-markers"]
|
||||
```
|
||||
|
||||
## Settings / config — pydantic-settings
|
||||
|
||||
Loads env vars and `.env` files into a Pydantic model. Replaces ad-hoc `os.environ.get(...)` everywhere.
|
||||
|
||||
```python
|
||||
from pydantic import Field
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", env_prefix="MYAPP_")
|
||||
|
||||
database_url: str
|
||||
api_key: str = Field(min_length=1)
|
||||
debug: bool = False
|
||||
|
||||
settings = Settings() # loads at import time; raises if any required var is missing
|
||||
```
|
||||
|
||||
## Logging — stdlib logging + rich handler
|
||||
|
||||
Stdlib `logging` is fine; it gets a face-lift from `rich.logging.RichHandler`.
|
||||
|
||||
```python
|
||||
import logging
|
||||
from rich.logging import RichHandler
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(message)s",
|
||||
datefmt="[%X]",
|
||||
handlers=[RichHandler(rich_tracebacks=True, show_path=False)],
|
||||
)
|
||||
log = logging.getLogger(__name__)
|
||||
log.info("ready")
|
||||
```
|
||||
|
||||
For structured logging in production, swap to `structlog` (separate dep). Don't roll your own.
|
||||
@@ -0,0 +1,268 @@
|
||||
# One-liner Scripts (PEP 723 + uv)
|
||||
|
||||
Self-contained Python scripts with declared dependencies, run with no environment setup. The combination eliminates the historical reason to write small tools in Go or Bash.
|
||||
|
||||
**Rule: EVERY `.py` script — even throwaway — MUST use PEP 723 inline metadata with the usage comment block.** No venv, no requirements.txt, no setup.py. The script IS the environment spec.
|
||||
|
||||
## The two patterns
|
||||
|
||||
### Pattern 1: inline `uv run` invocation
|
||||
|
||||
```bash
|
||||
uv run --with httpx2 --with rich python -c "
|
||||
import httpx2
|
||||
from rich import print
|
||||
print(httpx2.get('https://api.github.com').json())
|
||||
"
|
||||
```
|
||||
|
||||
Use for terminal one-shots that you don't want to save. `--with PKG` may be repeated.
|
||||
|
||||
### Pattern 2: PEP 723 script with shebang (THE CANONICAL PATTERN)
|
||||
|
||||
A regular `.py` file with metadata in a comment block. uv reads the metadata, materialises a disposable venv (cached), and runs the script.
|
||||
|
||||
```python
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# requires-python = ">=3.13"
|
||||
# dependencies = [
|
||||
# "httpx2[http2,brotli,zstd]",
|
||||
# "rich",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
# ─── How to run ───
|
||||
# 1. Install uv (if not installed):
|
||||
# curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
# 2. Run directly (no venv, no pip install needed):
|
||||
# uv run my_script.py
|
||||
# 3. Or make executable and run:
|
||||
# chmod +x my_script.py && ./my_script.py
|
||||
# ──────────────────
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx2
|
||||
from rich import print as rprint
|
||||
|
||||
|
||||
def main() -> None:
|
||||
with httpx2.Client(http2=True, follow_redirects=True) as client:
|
||||
resp = client.get("https://api.github.com")
|
||||
resp.raise_for_status()
|
||||
rprint(resp.json())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
```
|
||||
|
||||
### Mandatory elements
|
||||
|
||||
Every PEP 723 script MUST include these, in order:
|
||||
|
||||
1. **Shebang**: `#!/usr/bin/env -S uv run --script`
|
||||
2. **PEP 723 metadata block**: `# /// script` ... `# ///` with `requires-python` and `dependencies`
|
||||
3. **Usage comment block**: How to install uv + how to run the script. Copy the template above verbatim.
|
||||
4. **`from __future__ import annotations`**: Always first import.
|
||||
5. **`if __name__ == "__main__": main()`**: Entry point guard.
|
||||
|
||||
### The usage comment block (NON-NEGOTIABLE)
|
||||
|
||||
```python
|
||||
# ─── How to run ───
|
||||
# 1. Install uv (if not installed):
|
||||
# curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
# 2. Run directly (no venv, no pip install needed):
|
||||
# uv run <SCRIPT_NAME>.py [ARGS]
|
||||
# 3. Or make executable and run:
|
||||
# chmod +x <SCRIPT_NAME>.py && ./<SCRIPT_NAME>.py
|
||||
# ──────────────────
|
||||
```
|
||||
|
||||
Replace `<SCRIPT_NAME>` with the actual filename. Add argument descriptions if the script takes CLI args. This block goes immediately after the `# ///` closing line, before any imports.
|
||||
|
||||
**Why mandatory**: Anyone who receives this script — colleague, CI, future you — must know how to run it without reading docs. The comment IS the docs.
|
||||
|
||||
## Template generator
|
||||
|
||||
Use `scripts/new-script.py` to scaffold a new PEP 723 script with all boilerplate pre-filled:
|
||||
|
||||
```bash
|
||||
# Generate to temp directory (default)
|
||||
uv run scripts/new-script.py my_tool
|
||||
|
||||
# Generate to specific path
|
||||
uv run scripts/new-script.py my_tool --output ./scripts/my_tool.py
|
||||
|
||||
# With extra dependencies
|
||||
uv run scripts/new-script.py my_tool --deps "polars" "duckdb" "rich"
|
||||
```
|
||||
|
||||
## Common dependency sets
|
||||
|
||||
| Use case | Dependencies line |
|
||||
|---|---|
|
||||
| API client | `"httpx2[http2,brotli,zstd]"` |
|
||||
| Data processing | `"polars"`, `"duckdb"` |
|
||||
| CLI tool | `"typer"`, `"rich"` |
|
||||
| Web scraping | `"httpx2[http2,brotli,zstd]"`, `"selectolax"` |
|
||||
| File watcher | `"watchfiles"` |
|
||||
| JSON pretty | `"rich"` |
|
||||
| AI / LLM | `"pydantic-ai"`, `"httpx2[http2,brotli,zstd]"` |
|
||||
|
||||
## Real-world examples
|
||||
|
||||
### Fetch + print JSON
|
||||
|
||||
```python
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# requires-python = ">=3.13"
|
||||
# dependencies = [
|
||||
# "httpx2[http2,brotli,zstd]",
|
||||
# "rich",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
# ─── How to run ───
|
||||
# 1. Install uv: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
# 2. Run: uv run fetch_json.py https://api.github.com/repos/pydantic/httpx2
|
||||
# ──────────────────
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
import httpx2
|
||||
from rich import print as rprint
|
||||
|
||||
|
||||
def main() -> None:
|
||||
url = sys.argv[1] if len(sys.argv) > 1 else "https://api.github.com"
|
||||
with httpx2.Client(http2=True, follow_redirects=True) as client:
|
||||
resp = client.get(url)
|
||||
resp.raise_for_status()
|
||||
rprint(resp.json())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
```
|
||||
|
||||
### CSV → Parquet conversion
|
||||
|
||||
```python
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# requires-python = ">=3.13"
|
||||
# dependencies = [
|
||||
# "polars",
|
||||
# "typer",
|
||||
# "rich",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
# ─── How to run ───
|
||||
# 1. Install uv: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
# 2. Run: uv run csv2parquet.py input.csv output.parquet
|
||||
# ──────────────────
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
import typer
|
||||
from rich import print as rprint
|
||||
|
||||
|
||||
def main(input_path: Path, output_path: Path | None = None) -> None:
|
||||
"""Convert CSV to Parquet."""
|
||||
out = output_path or input_path.with_suffix(".parquet")
|
||||
df = pl.read_csv(input_path)
|
||||
df.write_parquet(out)
|
||||
rprint(f"[green]✓[/green] {input_path} → {out} ({len(df)} rows)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
typer.run(main)
|
||||
```
|
||||
|
||||
### Quick benchmark
|
||||
|
||||
```python
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# requires-python = ">=3.13"
|
||||
# dependencies = [
|
||||
# "httpx2[http2,brotli,zstd]",
|
||||
# "rich",
|
||||
# "anyio",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
# ─── How to run ───
|
||||
# 1. Install uv: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
# 2. Run: uv run bench.py https://api.example.com/health 50
|
||||
# ──────────────────
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
|
||||
import anyio
|
||||
import httpx2
|
||||
from rich import print as rprint
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
url = sys.argv[1] if len(sys.argv) > 1 else "https://api.github.com"
|
||||
n = int(sys.argv[2]) if len(sys.argv) > 2 else 20
|
||||
|
||||
limits = httpx2.Limits(max_connections=200, max_keepalive_connections=40, keepalive_expiry=30.0)
|
||||
timeout = httpx2.Timeout(connect=5.0, read=30.0, write=10.0, pool=10.0)
|
||||
transport = httpx2.AsyncHTTPTransport(
|
||||
http2=True, retries=3, limits=limits,
|
||||
socket_options=[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)],
|
||||
)
|
||||
|
||||
async with httpx2.AsyncClient(transport=transport, timeout=timeout, follow_redirects=True) as client:
|
||||
# warmup
|
||||
for _ in range(3):
|
||||
await client.get(url)
|
||||
|
||||
start = time.perf_counter()
|
||||
for _ in range(n):
|
||||
r = await client.get(url)
|
||||
assert r.status_code == 200
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
avg_ms = (elapsed / n) * 1000
|
||||
rprint(f"[bold]{url}[/bold]: {avg_ms:.1f}ms avg over {n} requests ({elapsed:.2f}s total, {r.http_version})")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
anyio.run(main)
|
||||
```
|
||||
|
||||
## Anti-patterns
|
||||
|
||||
| ❌ Don't | ✅ Do |
|
||||
|---|---|
|
||||
| `pip install httpx2 && python script.py` | `uv run script.py` |
|
||||
| `requirements.txt` alongside script | PEP 723 inline metadata |
|
||||
| `python -m venv .venv && ...` | `uv run --script` handles it |
|
||||
| Script without usage comment | Always include the "How to run" block |
|
||||
| `import asyncio; asyncio.run(main())` | `import anyio; anyio.run(main)` |
|
||||
| Bare `httpx2.AsyncClient()` | Full production defaults (see `references/httpx2-optimization.md`) |
|
||||
|
||||
## Sources
|
||||
|
||||
- PEP 723 - Inline script metadata: <https://peps.python.org/pep-0723/>
|
||||
- uv `run --script` docs: <https://docs.astral.sh/uv/guides/scripts/>
|
||||
- Original article: <https://www.cottongeeks.com/articles/2025-06-24-fun-with-uv-and-pep-723>
|
||||
- Simon Willison on one-shot Python tools: <https://simonwillison.net/2024/Dec/19/one-shot-python-tools/>
|
||||
@@ -0,0 +1,378 @@
|
||||
# orjson — When to Use, How to Integrate
|
||||
|
||||
`orjson` is the fastest JSON library on PyPI — written in Rust, 6–11× faster than stdlib `json` on serialization, 1.5–4× faster on deserialization. It also supports types the stdlib refuses to serialize: `datetime`, `date`, `UUID`, `numpy` arrays, `dataclass`, Pydantic models (via a small bridge).
|
||||
|
||||
This document covers the production patterns. **Not every project needs orjson.** The decision tree is in §1.
|
||||
|
||||
---
|
||||
|
||||
## 1. Decision tree — should you adopt orjson?
|
||||
|
||||
```
|
||||
Are you serializing/deserializing JSON in a hot path?
|
||||
├─ NO → stdlib `json` is fine. Stop here.
|
||||
└─ YES ↓
|
||||
|
||||
Is the project FastAPI?
|
||||
├─ YES ↓
|
||||
│
|
||||
│ Is your response body fully described by a Pydantic v2 model?
|
||||
│ ├─ YES → Use FastAPI's default JSON response (uses Pydantic's
|
||||
│ │ Rust-backed serializer; orjson saves nothing in this path).
|
||||
│ │ Adopt orjson only for *non-Pydantic* responses below.
|
||||
│ └─ NO → Use `ORJSONResponse` for endpoints that return dicts,
|
||||
│ lists, or arbitrary structures.
|
||||
│
|
||||
└─ NOT FastAPI ↓
|
||||
|
||||
Are you serializing Pydantic v2 models repeatedly?
|
||||
├─ YES → Use `model.model_dump_json()` directly — backed by pydantic-core
|
||||
│ (Rust), within ~10% of orjson on the same payload, and respects
|
||||
│ every Pydantic feature (computed fields, aliases, validators).
|
||||
└─ NO ↓
|
||||
|
||||
Are you serializing dicts / lists / dataclasses / datetime / UUID?
|
||||
├─ YES → orjson is the right answer.
|
||||
└─ NO → stdlib `json`.
|
||||
```
|
||||
|
||||
**The crucial 2026 fact**: with Pydantic v2's `model_dump_json()`, **Pydantic-shaped responses no longer need orjson**. Adopt orjson where you are still going through `dict` / `list` / `dataclass`.
|
||||
|
||||
---
|
||||
|
||||
## 2. Install
|
||||
|
||||
```toml
|
||||
# pyproject.toml
|
||||
dependencies = [
|
||||
"orjson>=3.10",
|
||||
]
|
||||
```
|
||||
|
||||
orjson wheels are published for every major CPython version and platform (macOS, Linux glibc/musl, Windows, ARM64). No compilation step on install.
|
||||
|
||||
---
|
||||
|
||||
## 3. Basic usage
|
||||
|
||||
```python
|
||||
import orjson
|
||||
|
||||
# Serialization — returns bytes, not str
|
||||
raw: bytes = orjson.dumps({"hello": "world", "ts": datetime.now(UTC)})
|
||||
|
||||
# Deserialization
|
||||
data = orjson.loads(raw)
|
||||
```
|
||||
|
||||
Two things to internalize:
|
||||
|
||||
1. **`orjson.dumps` returns `bytes`**, not `str`. Stdlib `json.dumps` returns `str`. This is by design — most JSON destinations (sockets, files in binary mode, HTTP bodies) want bytes anyway, and skipping the encode/decode round trip is part of the speedup.
|
||||
2. **No `indent` arg.** orjson supports `OPT_INDENT_2` (and only 2-space indent) via flags. If you need other indentation, use stdlib `json`.
|
||||
|
||||
---
|
||||
|
||||
## 4. The option flags you actually use
|
||||
|
||||
```python
|
||||
import orjson
|
||||
|
||||
orjson.dumps(
|
||||
payload,
|
||||
option=(
|
||||
orjson.OPT_NAIVE_UTC # treat naive datetimes as UTC (recommended)
|
||||
| orjson.OPT_UTC_Z # render UTC as "...Z" instead of "+00:00"
|
||||
| orjson.OPT_SERIALIZE_NUMPY # serialize numpy arrays natively
|
||||
| orjson.OPT_SERIALIZE_DATACLASS # serialize @dataclass instances
|
||||
| orjson.OPT_NON_STR_KEYS # allow int / UUID / datetime dict keys
|
||||
# | orjson.OPT_SORT_KEYS # only when you need deterministic output
|
||||
# | orjson.OPT_INDENT_2 # only for human-readable output (slower)
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
Each flag is opt-in for a reason — orjson defaults to spec-strict JSON.
|
||||
|
||||
The flag combination above is a sensible "production default" for application code. The `OPT_NAIVE_UTC | OPT_UTC_Z` pair is especially important: it produces RFC 3339 timestamps that every parser on earth accepts.
|
||||
|
||||
---
|
||||
|
||||
## 5. orjson + FastAPI
|
||||
|
||||
### 5.1 The legacy pattern: `ORJSONResponse`
|
||||
|
||||
```python
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import ORJSONResponse
|
||||
|
||||
app = FastAPI(default_response_class=ORJSONResponse)
|
||||
|
||||
@app.get("/items")
|
||||
async def get_items() -> dict[str, list[dict[str, int]]]:
|
||||
return {"items": [{"id": i, "qty": i * 2} for i in range(1000)]}
|
||||
```
|
||||
|
||||
`default_response_class=ORJSONResponse` swaps the global JSON encoder for orjson. **This affects only the response body serialization**, not request parsing — for request parsing, FastAPI still uses Pydantic.
|
||||
|
||||
### 5.2 The 2026 reality — Pydantic v2 vs orjson
|
||||
|
||||
With FastAPI 0.100+ on Pydantic v2:
|
||||
|
||||
- If your response is annotated with a Pydantic model, FastAPI calls `model_dump_json()` directly. **orjson is bypassed** even with `default_response_class=ORJSONResponse`, because the Pydantic serializer is already Rust-backed.
|
||||
- If your response is a raw `dict` / `list` / Python object, `ORJSONResponse` does kick in and saves real time.
|
||||
|
||||
The benchmark in `tiangolo/fastapi#11728` (Apr 2024) showed `model_dump_json()` is ~10–15% faster than `ORJSONResponse + model_dump()` for Pydantic-shaped responses. The shape of the data matters; on mixed-shape APIs, keep `ORJSONResponse` as the default and trust Pydantic's path for typed responses.
|
||||
|
||||
### 5.3 Recommended setup
|
||||
|
||||
```python
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import ORJSONResponse
|
||||
|
||||
app = FastAPI(
|
||||
default_response_class=ORJSONResponse, # benefits dict/list returns
|
||||
# Pydantic-typed returns automatically use pydantic-core serialization
|
||||
)
|
||||
```
|
||||
|
||||
**Do NOT** wrap Pydantic models manually:
|
||||
|
||||
```python
|
||||
# BAD — defeats Pydantic's optimized path
|
||||
@app.get("/users/{id}", response_class=ORJSONResponse)
|
||||
async def get_user(id: int) -> ORJSONResponse:
|
||||
user = await fetch_user(id)
|
||||
return ORJSONResponse(content=user.model_dump()) # extra dict trip
|
||||
|
||||
# GOOD — let FastAPI serialize the model
|
||||
@app.get("/users/{id}")
|
||||
async def get_user(id: int) -> User:
|
||||
return await fetch_user(id)
|
||||
```
|
||||
|
||||
### 5.4 Streaming responses
|
||||
|
||||
`ORJSONResponse` does not stream — it buffers the whole response. For SSE, NDJSON, or chunked JSON, use `StreamingResponse` and call `orjson.dumps` per chunk:
|
||||
|
||||
```python
|
||||
from fastapi.responses import StreamingResponse
|
||||
import orjson
|
||||
|
||||
async def ndjson_stream():
|
||||
async for row in fetch_rows():
|
||||
yield orjson.dumps(row) + b"\n"
|
||||
|
||||
@app.get("/export")
|
||||
async def export():
|
||||
return StreamingResponse(ndjson_stream(), media_type="application/x-ndjson")
|
||||
```
|
||||
|
||||
This is where orjson shines — per-chunk serialization in a tight loop, zero buffering.
|
||||
|
||||
---
|
||||
|
||||
## 6. orjson + Pydantic v2 (no FastAPI)
|
||||
|
||||
When you have a Pydantic model and want orjson's output for non-FastAPI contexts:
|
||||
|
||||
```python
|
||||
from pydantic import BaseModel
|
||||
import orjson
|
||||
|
||||
class User(BaseModel):
|
||||
id: int
|
||||
email: str
|
||||
created: datetime
|
||||
|
||||
user = User(id=1, email="a@b.com", created=datetime.now(UTC))
|
||||
|
||||
# Option A — Pydantic's built-in Rust serializer (USE THIS by default)
|
||||
raw: bytes = user.model_dump_json().encode()
|
||||
# 2026: ~1.2× faster than orjson on the same payload, supports
|
||||
# every Pydantic feature (aliases, computed fields, json_schema_extra, etc.)
|
||||
|
||||
# Option B — orjson bridge for cases Pydantic does not cover
|
||||
raw: bytes = orjson.dumps(
|
||||
user,
|
||||
default=lambda obj: obj.model_dump() if isinstance(obj, BaseModel) else None,
|
||||
)
|
||||
# Useful when serializing nested non-Pydantic structures that contain
|
||||
# BaseModels — e.g. a list of dicts that each may contain a BaseModel.
|
||||
```
|
||||
|
||||
For routine "serialize one Pydantic model to JSON", `model_dump_json()` wins on speed AND feature parity. Reach for orjson only at the *container* level (a dict of mixed types).
|
||||
|
||||
### Custom `default=` callback — the universal extension point
|
||||
|
||||
```python
|
||||
import orjson
|
||||
from decimal import Decimal
|
||||
from pydantic import BaseModel
|
||||
|
||||
def _default(obj):
|
||||
if isinstance(obj, BaseModel):
|
||||
return obj.model_dump()
|
||||
if isinstance(obj, Decimal):
|
||||
return str(obj)
|
||||
if isinstance(obj, set):
|
||||
return list(obj)
|
||||
raise TypeError(f"orjson: cannot serialize {type(obj).__name__}")
|
||||
|
||||
orjson.dumps(payload, default=_default, option=orjson.OPT_NAIVE_UTC | orjson.OPT_UTC_Z)
|
||||
```
|
||||
|
||||
The `default=` callback runs once per unrecognized type, then orjson caches the path. Performance impact on subsequent calls is negligible.
|
||||
|
||||
---
|
||||
|
||||
## 7. Caching, queues, logging — the prime orjson use cases
|
||||
|
||||
These are where orjson pays off most clearly because there is no Pydantic in the loop:
|
||||
|
||||
### Redis cache
|
||||
|
||||
```python
|
||||
import orjson
|
||||
import redis.asyncio as redis
|
||||
|
||||
r = redis.from_url("redis://localhost")
|
||||
|
||||
async def set_cache(key: str, value: dict) -> None:
|
||||
await r.set(key, orjson.dumps(value), ex=3600)
|
||||
|
||||
async def get_cache(key: str) -> dict | None:
|
||||
raw = await r.get(key)
|
||||
return orjson.loads(raw) if raw else None
|
||||
```
|
||||
|
||||
`orjson` over stdlib `json` here saves ~5–10× on the serialize step for typical cache payloads. Multiply by request rate.
|
||||
|
||||
### Task queue payloads (Celery, RQ, dramatiq)
|
||||
|
||||
```python
|
||||
# Celery custom serializer
|
||||
from kombu.serialization import register
|
||||
import orjson
|
||||
|
||||
def _orjson_dumps(obj):
|
||||
return orjson.dumps(obj, option=orjson.OPT_NAIVE_UTC | orjson.OPT_UTC_Z).decode()
|
||||
|
||||
def _orjson_loads(s):
|
||||
return orjson.loads(s)
|
||||
|
||||
register("orjson", _orjson_dumps, _orjson_loads,
|
||||
content_type="application/x-orjson",
|
||||
content_encoding="utf-8")
|
||||
```
|
||||
|
||||
Same speedup, applied to every task payload encode/decode.
|
||||
|
||||
### Structured logging (structlog, custom slog)
|
||||
|
||||
```python
|
||||
import structlog
|
||||
import orjson
|
||||
|
||||
structlog.configure(
|
||||
processors=[
|
||||
structlog.processors.TimeStamper(fmt="iso"),
|
||||
structlog.processors.add_log_level,
|
||||
structlog.processors.JSONRenderer(serializer=orjson.dumps),
|
||||
],
|
||||
)
|
||||
```
|
||||
|
||||
structlog's `JSONRenderer` accepts any callable; orjson is the obvious default. Logging hot paths benefit dramatically — every log line at info level becomes ~5× cheaper to render.
|
||||
|
||||
---
|
||||
|
||||
## 8. Gotchas
|
||||
|
||||
### `orjson.dumps` returns bytes, not str
|
||||
|
||||
```python
|
||||
# BAD — concatenating bytes and str
|
||||
log.info("payload: " + orjson.dumps(data)) # TypeError
|
||||
|
||||
# GOOD
|
||||
log.info("payload: %s", orjson.dumps(data).decode())
|
||||
# or
|
||||
log.info("payload: %s", orjson.dumps(data)) # let the formatter handle it
|
||||
```
|
||||
|
||||
### No `cls=` argument for custom encoders
|
||||
|
||||
orjson uses `default=` only. If you have a custom `JSONEncoder` subclass from stdlib `json`, port its `default()` method to a `default=` callable.
|
||||
|
||||
### Subclasses of `dict` / `list` are NOT serialized as their parent
|
||||
|
||||
```python
|
||||
class StrictDict(dict): ...
|
||||
d = StrictDict({"k": "v"})
|
||||
|
||||
import json
|
||||
json.dumps(d) # OK — stdlib walks subclasses
|
||||
orjson.dumps(d) # TypeError — orjson is strict by design
|
||||
orjson.dumps(d, option=orjson.OPT_PASSTHROUGH_SUBCLASS) # then route via default=
|
||||
```
|
||||
|
||||
Set `OPT_PASSTHROUGH_SUBCLASS` and handle the subclass in `default=`. The design discourages accidental subclass usage that breaks elsewhere.
|
||||
|
||||
### `int` overflow
|
||||
|
||||
orjson refuses to encode integers larger than 2⁵³ - 1 by default (the IEEE-754 double-precision safe-integer limit — what JavaScript can round-trip). For larger ints, opt in:
|
||||
|
||||
```python
|
||||
orjson.dumps(huge_int, option=orjson.OPT_STRICT_INTEGER) # error
|
||||
orjson.dumps(huge_int) # default — int is encoded as JSON number
|
||||
# JavaScript clients lose precision past 2^53; consider sending as string
|
||||
```
|
||||
|
||||
This is more spec-strict than stdlib `json`, which silently emits ints of any size.
|
||||
|
||||
### Timezone-naive datetimes
|
||||
|
||||
By default, orjson treats naive `datetime` as the system local timezone — almost never what you want. **Always set `OPT_NAIVE_UTC`** to treat naive datetimes as UTC, or use timezone-aware datetimes (which is the better long-term habit).
|
||||
|
||||
---
|
||||
|
||||
## 9. Benchmark — should I actually adopt this?
|
||||
|
||||
The numbers below are 2024–2026 averages from `tiangolo/fastapi#11728` and orjson's own benchmark suite, on Python 3.13, modern x86_64:
|
||||
|
||||
| Payload | stdlib `json` | `orjson` | `model_dump_json()` (Pydantic v2) |
|
||||
|---|---|---|---|
|
||||
| Small dict (100 fields) | 1.0× | **8×** | n/a |
|
||||
| List of 10k dicts | 1.0× | **11×** | n/a |
|
||||
| Pydantic model with 20 fields | 1.0× (after `model_dump()`) | 5× (with `default=` bridge) | **6×** |
|
||||
| Datetime-heavy payload | 1.0× (after manual ISO conv) | **9×** | 6× |
|
||||
| numpy array (1M floats) | impossible without manual conv | **20×** vs json+tolist | n/a |
|
||||
|
||||
The takeaways:
|
||||
|
||||
- For raw dict/list/datetime, **orjson is dramatically faster**.
|
||||
- For Pydantic models, **`model_dump_json()` is already faster than orjson+bridge**.
|
||||
- For numpy, orjson is the only sane choice.
|
||||
|
||||
In production, the actual measured win on a FastAPI app with mixed payloads is typically 5–15% reduction in p99 latency. Worth the one-line `default_response_class=ORJSONResponse` switch.
|
||||
|
||||
---
|
||||
|
||||
## 10. When NOT to adopt orjson
|
||||
|
||||
- The codebase is small, JSON is not a bottleneck, and you have no measured perf concern.
|
||||
- You depend on stdlib `json`'s `cls=` arg or its lax tolerance for non-spec input (NaN, Infinity, comments).
|
||||
- You need pretty-printed JSON with custom indent — orjson only supports 2-space indent via the flag.
|
||||
- You need pure-Python portability (e.g., MicroPython, no-wheel platforms) — orjson is a compiled Rust extension.
|
||||
|
||||
If the choice is "add a dependency that does 5–10× the speed on serialization for free", the answer is almost always yes. The "almost" is in the bullets above.
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- orjson: https://github.com/ijl/orjson
|
||||
- Pydantic v2 `model_dump_json`: https://docs.pydantic.dev/latest/concepts/serialization/#modelmodel_dump_json
|
||||
- FastAPI `ORJSONResponse`: https://fastapi.tiangolo.com/advanced/custom-response/#use-orjsonresponse
|
||||
- "FastAPI + orjson vs Pydantic v2" benchmark: https://github.com/fastapi/fastapi/discussions/11728
|
||||
- structlog JSON rendering: https://www.structlog.org/en/stable/api.html#structlog.processors.JSONRenderer
|
||||
@@ -0,0 +1,285 @@
|
||||
# PydanticAI Reference (v1.x, 2026)
|
||||
|
||||
> Canonical patterns for wiring PydanticAI agents. Target: production usage, late-2025 / 2026.
|
||||
> Source: [ai.pydantic.dev](https://ai.pydantic.dev) and [pydantic/pydantic-ai@`cad9569`](https://github.com/pydantic/pydantic-ai/blob/cad956910079737ea0886b50cef15777208f92e6).
|
||||
|
||||
---
|
||||
|
||||
## 1. Agent Constructor
|
||||
|
||||
```python
|
||||
from pydantic_ai import Agent
|
||||
|
||||
agent = Agent(
|
||||
'openai:gpt-5.2', # model (str | Model | None)
|
||||
output_type=MyOutputModel, # structured output type; default=str
|
||||
instructions='You are a...', # static or callable instructions
|
||||
system_prompt='Be concise.', # static system prompt(s)
|
||||
deps_type=MyDeps, # dependency type for type-checking only
|
||||
name='my-agent', # optional, inferred from var name if omitted
|
||||
retries=1, # default retries for tools + output validation
|
||||
output_retries=None, # override retries for output validation only
|
||||
tools=[my_tool], # list of Tool objects or plain functions
|
||||
defer_model_check=False, # set True to skip env-var check at init time
|
||||
end_strategy='early', # 'early' | 'graceful' | 'exhaustive'
|
||||
)
|
||||
```
|
||||
|
||||
**Breaking change (v1.88.0)**: `result_type` was renamed to `output_type`. Use `output_type`.
|
||||
|
||||
---
|
||||
|
||||
## 2. Model Strings
|
||||
|
||||
Format: `provider:model-name`. The framework infers the provider from the prefix.
|
||||
|
||||
| Provider prefix | Example |
|
||||
|---|---|
|
||||
| `openai:` | `'openai:gpt-5.2'`, `'openai:gpt-4o'` |
|
||||
| `anthropic:` | `'anthropic:claude-sonnet-4-6'`, `'anthropic:claude-opus-4-1'` |
|
||||
| `google-gla:` | `'google-gla:gemini-3-flash-preview'` |
|
||||
| `google-vertex:` | `'google-vertex:gemini-3-pro-preview'` |
|
||||
| `bedrock:` | `'bedrock:anthropic.claude-sonnet-4-6'` |
|
||||
| `xai:` / `grok:` | `'xai:grok-3'`, `'grok:grok-3-fast'` |
|
||||
| `deepseek:` | `'deepseek:deepseek-chat'` |
|
||||
| `cohere:` | `'cohere:command-r-08-2024'` |
|
||||
| `gateway/...` | `'gateway/openai:gpt-5.2'` (PydanticAI Gateway) |
|
||||
|
||||
Model can also be omitted at construction and passed per-run: `agent.run(prompt, model='openai:gpt-5.2')`.
|
||||
|
||||
---
|
||||
|
||||
## 3. Tools
|
||||
|
||||
### Decorator syntax
|
||||
|
||||
```python
|
||||
from pydantic_ai import Agent, RunContext
|
||||
|
||||
agent = Agent('openai:gpt-5.2', deps_type=str)
|
||||
|
||||
@agent.tool # default: receives RunContext as first arg
|
||||
async def greet(ctx: RunContext[str], name: str) -> str:
|
||||
return f"Hello {ctx.deps}, {name}!"
|
||||
|
||||
@agent.tool_plain # no context needed
|
||||
async def roll_dice(sides: int) -> int:
|
||||
import random
|
||||
return random.randint(1, sides)
|
||||
```
|
||||
|
||||
### `RunContext[Deps]`
|
||||
|
||||
First parameter of `@agent.tool` functions. Carries:
|
||||
|
||||
- `ctx.deps` — the dependency instance
|
||||
- `ctx.model` — the model being used
|
||||
- `ctx.usage` — token usage so far
|
||||
- `ctx.messages` — conversation history
|
||||
- `ctx.retry` / `ctx.max_retries` — current retry count
|
||||
- `ctx.agent` — the running agent instance
|
||||
|
||||
Use `@agent.tool_plain` when the tool does **not** need any of the above.
|
||||
|
||||
---
|
||||
|
||||
## 4. Structured Output
|
||||
|
||||
Pass a Pydantic `BaseModel` (or `bool`, `int`, `list[str]`, etc.) as `output_type`. The result is accessed via `.output`.
|
||||
|
||||
```python
|
||||
from pydantic import BaseModel
|
||||
from pydantic_ai import Agent
|
||||
|
||||
class City(BaseModel):
|
||||
name: str
|
||||
country: str
|
||||
population_millions: float
|
||||
|
||||
agent = Agent('openai:gpt-5.2', output_type=City)
|
||||
result = agent.run_sync('Tell me about Tokyo')
|
||||
print(result.output) # City(name='Tokyo', country='Japan', ...)
|
||||
print(result.output.name) # 'Tokyo'
|
||||
```
|
||||
|
||||
**Note**: `result.data` was renamed; the canonical accessor is `result.output`.
|
||||
|
||||
---
|
||||
|
||||
## 5. Async vs Sync
|
||||
|
||||
| Method | Mode | Returns |
|
||||
|---|---|---|
|
||||
| `await agent.run(prompt, ...)` | async | `AgentRunResult[OutputDataT]` |
|
||||
| `agent.run_sync(prompt, ...)` | sync | `AgentRunResult[OutputDataT]` |
|
||||
| `async with agent.run_stream(prompt, ...) as response:` | async streaming | `StreamedRunResult` |
|
||||
|
||||
```python
|
||||
# Sync
|
||||
result = agent.run_sync('What is the capital of Italy?')
|
||||
print(result.output)
|
||||
|
||||
# Async
|
||||
result = await agent.run('What is the capital of France?')
|
||||
print(result.output)
|
||||
|
||||
# Streaming
|
||||
async with agent.run_stream('What is the capital of the UK?') as response:
|
||||
async for text in response.stream_text():
|
||||
print(text, end='')
|
||||
# After streaming finishes:
|
||||
print(response.output)
|
||||
```
|
||||
|
||||
`run_sync()` is a convenience wrapper over `loop.run_until_complete(self.run(...))`. Do not use it inside an active async context.
|
||||
|
||||
---
|
||||
|
||||
## 6. Dependencies
|
||||
|
||||
Use a `@dataclass` container, pass the **type** to `deps_type`, and pass an **instance** to `deps` at run time.
|
||||
|
||||
```python
|
||||
from dataclasses import dataclass
|
||||
import httpx
|
||||
from pydantic_ai import Agent, RunContext
|
||||
|
||||
@dataclass
|
||||
class Deps:
|
||||
api_key: str
|
||||
http_client: httpx.AsyncClient
|
||||
|
||||
agent = Agent(
|
||||
'openai:gpt-5.2',
|
||||
deps_type=Deps,
|
||||
)
|
||||
|
||||
@agent.tool
|
||||
async def fetch_data(ctx: RunContext[Deps], endpoint: str) -> str:
|
||||
r = await ctx.deps.http_client.get(
|
||||
endpoint,
|
||||
headers={'Authorization': f'Bearer {ctx.deps.api_key}'},
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.text
|
||||
|
||||
async def main():
|
||||
async with httpx.AsyncClient() as client:
|
||||
deps = Deps(api_key='sk-...', http_client=client)
|
||||
result = await agent.run('Get /users', deps=deps)
|
||||
print(result.output)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. Error Types & Retrying from a Tool
|
||||
|
||||
```python
|
||||
from pydantic_ai import Agent, ModelRetry, UnexpectedModelBehavior, capture_run_messages
|
||||
|
||||
agent = Agent('openai:gpt-5.2', retries=3)
|
||||
|
||||
@agent.tool_plain
|
||||
def calc_volume(size: int) -> int:
|
||||
if size == 42:
|
||||
return size ** 3
|
||||
raise ModelRetry('Please try again with size 42.')
|
||||
|
||||
with capture_run_messages() as messages:
|
||||
try:
|
||||
result = agent.run_sync('Get the volume of a box with size 6.')
|
||||
except UnexpectedModelBehavior as e:
|
||||
print('Error:', e) # "Tool 'calc_volume' exceeded max retries count of 3"
|
||||
print('Cause:', e.__cause__) # ModelRetry('Please try again...')
|
||||
print('Messages:', messages)
|
||||
```
|
||||
|
||||
- **`ModelRetry`** — raise from a tool, output validator, or capability hook to ask the model to retry.
|
||||
- **`UnexpectedModelBehavior`** — raised when the retry limit is exceeded or the model API returns an unrecoverable error.
|
||||
- **`capture_run_messages()`** — context manager that records all messages exchanged during a run for debugging.
|
||||
|
||||
---
|
||||
|
||||
## 8. Logfire Integration
|
||||
|
||||
One-line setup if the `logfire` extra is installed (included in the default `pydantic-ai` package):
|
||||
|
||||
```python
|
||||
import logfire
|
||||
|
||||
logfire.configure() # reads token from .logfire directory
|
||||
logfire.instrument_pydantic_ai() # auto-traces all agent runs
|
||||
```
|
||||
|
||||
Alternatively, set `instrument=True` on the agent:
|
||||
|
||||
```python
|
||||
agent = Agent('openai:gpt-5.2', instrument=True)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9. Minimal Complete Snippets
|
||||
|
||||
### (a) Basic agent with structured output
|
||||
|
||||
```python
|
||||
from pydantic import BaseModel
|
||||
from pydantic_ai import Agent
|
||||
|
||||
class City(BaseModel):
|
||||
name: str
|
||||
country: str
|
||||
|
||||
agent = Agent('openai:gpt-5.2', output_type=City)
|
||||
result = agent.run_sync('Tell me about Paris')
|
||||
print(result.output) # City(name='Paris', country='France')
|
||||
```
|
||||
|
||||
### (b) Agent with tools and dependencies
|
||||
|
||||
```python
|
||||
from dataclasses import dataclass
|
||||
from pydantic_ai import Agent, RunContext
|
||||
|
||||
@dataclass
|
||||
class Deps:
|
||||
api_key: str
|
||||
|
||||
agent = Agent('openai:gpt-5.2', deps_type=Deps)
|
||||
|
||||
@agent.tool
|
||||
async def get_secret(ctx: RunContext[Deps], code: str) -> str:
|
||||
if code == '1234':
|
||||
return f'secret-for-{ctx.deps.api_key}'
|
||||
return 'wrong code'
|
||||
|
||||
result = agent.run_sync('My code is 1234', deps=Deps(api_key='sk-abc'))
|
||||
print(result.output)
|
||||
```
|
||||
|
||||
### (c) Async streaming
|
||||
|
||||
```python
|
||||
import anyio
|
||||
from pydantic_ai import Agent
|
||||
|
||||
agent = Agent('openai:gpt-5.2')
|
||||
|
||||
async def main() -> None:
|
||||
async with agent.run_stream('Write a haiku about Python') as response:
|
||||
async for text in response.stream_text():
|
||||
print(text, end='')
|
||||
print('\n---')
|
||||
print('Final:', response.output)
|
||||
|
||||
anyio.run(main)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Version Notes
|
||||
|
||||
- **V1** reached API stability in September 2025. Breaking changes are reserved for V2 (earliest April 2026).
|
||||
- **v1.88.0** renamed `result_type` → `output_type` and `result_tool_name` / `result_tool_description` were removed. Use `output_type`.
|
||||
- The canonical accessor for run results is `result.output` (not `result.data`).
|
||||
@@ -0,0 +1,232 @@
|
||||
# Strict pyproject.toml (basedpyright + ruff + uv)
|
||||
|
||||
The canonical "super strict but sane" config for modern Python projects. Copy-paste, then add your own dependencies.
|
||||
|
||||
## Bootstrap
|
||||
|
||||
```bash
|
||||
# Application
|
||||
uv init --app myproject
|
||||
cd myproject
|
||||
|
||||
# Library (publishable to PyPI)
|
||||
uv init --lib mylibrary
|
||||
cd mylibrary
|
||||
|
||||
# Add dev tools
|
||||
uv add --dev basedpyright ruff pytest
|
||||
```
|
||||
|
||||
`uv init` creates `pyproject.toml`, `.python-version`, and `src/` layout. Replace its `pyproject.toml` `[tool.*]` sections with the block below.
|
||||
|
||||
## The full pyproject.toml
|
||||
|
||||
```toml
|
||||
[project]
|
||||
name = "myproject"
|
||||
version = "0.1.0"
|
||||
description = "..."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.13"
|
||||
dependencies = []
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"basedpyright>=1.21",
|
||||
"ruff>=0.8",
|
||||
"pytest>=8",
|
||||
"pytest-cov>=5",
|
||||
]
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# basedpyright - typeCheckingMode = "all" sets every report flag to error
|
||||
# Source: https://docs.basedpyright.com/latest/configuration/config-files/
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
[tool.basedpyright]
|
||||
typeCheckingMode = "all"
|
||||
pythonVersion = "3.13"
|
||||
pythonPlatform = "All" # default in basedpyright; explicit for clarity
|
||||
include = ["src", "tests"]
|
||||
exclude = ["**/__pycache__", "**/.venv", "**/build", "**/dist"]
|
||||
|
||||
# Strict enforcement extras (most are already "error" under "all" mode,
|
||||
# but listing them explicitly documents the intent)
|
||||
reportUnusedCallResult = "warning" # flag ignored return values
|
||||
reportUnnecessaryTypeIgnoreComment = "error" # stale type: ignore comments must die
|
||||
reportUnusedVariable = "error" # unused variables are errors
|
||||
reportMissingParameterType = "error" # every parameter must have a type
|
||||
reportMissingReturnType = "error" # every function must declare its return type
|
||||
reportPrivateUsage = "error" # respect _private convention
|
||||
|
||||
# Optional: gradual adoption baseline
|
||||
# baselineFile = "./.basedpyright/baseline.json"
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# ruff - select = ["ALL"] enables every rule, then we ignore the
|
||||
# small set that conflicts with the formatter or is not useful.
|
||||
# Source: https://docs.astral.sh/ruff/linter/#rule-selection
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
[tool.ruff]
|
||||
target-version = "py313"
|
||||
line-length = 88 # ruff/black default; 100 or 120 also fine
|
||||
src = ["src", "tests"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["ALL"]
|
||||
ignore = [
|
||||
# Formatter conflicts (ruff itself tells you to ignore these)
|
||||
"COM812", # missing trailing comma
|
||||
"ISC001", # implicit string concat
|
||||
# Docstyle conflicts (pick D211 over D203, D212 over D213)
|
||||
"D203",
|
||||
"D213",
|
||||
# Project-specific noise
|
||||
"CPY001", # missing copyright notice
|
||||
"FBT001", # boolean positional arg in def
|
||||
"FBT002", # boolean positional default in def
|
||||
"TD002", # missing TODO author
|
||||
"TD003", # missing TODO link
|
||||
"FIX002", # line contains TODO (TODOs are allowed)
|
||||
]
|
||||
fixable = ["ALL"]
|
||||
unfixable = []
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"tests/**/*.py" = [
|
||||
"S101", # `assert` is the entire point of pytest
|
||||
"ARG", # unused args (fixtures appear unused)
|
||||
"PLR2004", # magic numbers in test data
|
||||
"SLF001", # tests need access to private members
|
||||
"D", # docstrings not required in tests
|
||||
]
|
||||
"scripts/**/*.py" = [
|
||||
"T201", # `print` allowed in scripts
|
||||
"INP001", # implicit namespace package
|
||||
]
|
||||
|
||||
[tool.ruff.lint.pydocstyle]
|
||||
convention = "google" # or "numpy" / "pep257"
|
||||
|
||||
[tool.ruff.lint.flake8-bugbear]
|
||||
# typer / fastapi rely on call-as-default for parameter metadata.
|
||||
# Without this, ruff B008 ("function call in default") fires on every typer/fastapi route.
|
||||
extend-immutable-calls = [
|
||||
"typer.Argument",
|
||||
"typer.Option",
|
||||
"fastapi.Depends",
|
||||
"fastapi.Query",
|
||||
"fastapi.Path",
|
||||
"fastapi.Body",
|
||||
"fastapi.Header",
|
||||
"fastapi.Cookie",
|
||||
"fastapi.File",
|
||||
"fastapi.Form",
|
||||
]
|
||||
|
||||
[tool.ruff.format]
|
||||
quote-style = "double"
|
||||
indent-style = "space"
|
||||
docstring-code-format = true
|
||||
docstring-code-line-length = "dynamic"
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# pytest
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
[tool.pytest.ini_options]
|
||||
minversion = "8.0"
|
||||
testpaths = ["tests"]
|
||||
addopts = [
|
||||
"-ra",
|
||||
"--strict-config",
|
||||
"--strict-markers",
|
||||
]
|
||||
filterwarnings = ["error"]
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# coverage
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
[tool.coverage.run]
|
||||
source = ["src"]
|
||||
branch = true
|
||||
|
||||
[tool.coverage.report]
|
||||
exclude_lines = [
|
||||
"pragma: no cover",
|
||||
"if TYPE_CHECKING:",
|
||||
"if typing.TYPE_CHECKING:",
|
||||
"raise NotImplementedError",
|
||||
"@(abc\\.)?abstractmethod",
|
||||
]
|
||||
```
|
||||
|
||||
## Why these settings
|
||||
|
||||
### basedpyright `typeCheckingMode = "all"`
|
||||
|
||||
basedpyright's modes, strictest first:
|
||||
|
||||
| Mode | Behavior |
|
||||
|---|---|
|
||||
| `"all"` | Every diagnostic at `error` |
|
||||
| `"recommended"` | Same rules; less severe ones at `warning`; `failOnWarnings = true` makes CI still fail |
|
||||
| `"strict"` | pyright's strict mode |
|
||||
| `"standard"` | Default |
|
||||
| `"basic"` / `"off"` | Loose / disabled |
|
||||
|
||||
`"all"` enables basedpyright-exclusive rules pyright lacks: `reportImplicitOverride`, `reportImplicitStringConcatenation`, `reportIncompatibleUnannotatedOverride`, `reportUnannotatedClassAttribute`. No need to opt-in to additional flags.
|
||||
|
||||
`pythonPlatform = "All"` is basedpyright's default (better than pyright's host-OS default) - it errors on platform-specific imports that fail on other OSes.
|
||||
|
||||
### ruff `select = ["ALL"]`
|
||||
|
||||
The official docs say *"Use ALL with discretion. Enabling ALL will implicitly enable new rules whenever you upgrade."* For a strict skill that is the intended behavior - every new ruff rule should be considered an error until you justify ignoring it.
|
||||
|
||||
The minimal ignore set:
|
||||
|
||||
| Rule | Reason |
|
||||
|---|---|
|
||||
| `COM812`, `ISC001` | Conflict with `ruff format` (ruff itself documents this) |
|
||||
| `D203` vs `D211`, `D213` vs `D212` | Mutually-exclusive docstring conventions; pick the modern one |
|
||||
| `CPY001` | Most projects don't need a copyright header on every file |
|
||||
| `FBT001`, `FBT002` | Boolean flags are ergonomic for CLI/typer; ban makes typer awkward |
|
||||
| `TD002`, `TD003`, `FIX002` | TODOs without a JIRA link are fine in solo / internal code |
|
||||
|
||||
`ANN101` and `ANN102` were **removed in ruff 0.8.0** (Nov 2024). Do NOT include them in `ignore` - ruff errors on unknown rule codes.
|
||||
|
||||
`per-file-ignores` for `tests/**` is the standard pattern from real-world repos like `community-of-python/auto-typing-final` and `Preston-Landers/concurrent-log-handler`.
|
||||
|
||||
## CI gate
|
||||
|
||||
```bash
|
||||
# In CI, fail on any violation:
|
||||
uv run basedpyright
|
||||
uv run ruff check
|
||||
uv run ruff format --check
|
||||
uv run pytest
|
||||
```
|
||||
|
||||
A single `make ci` target combining the four works fine.
|
||||
|
||||
## Enforcement summary
|
||||
|
||||
The config above, combined with `scripts/check-no-excuse-rules.py`, enforces:
|
||||
|
||||
| What | How |
|
||||
|---|---|
|
||||
| Exhaustive match | basedpyright `all` mode + `assert_never` |
|
||||
| No `Any` | basedpyright `all` mode + script `cast-any` rule |
|
||||
| Ignored return values | `reportUnusedCallResult = "warning"` |
|
||||
| Immutable default | Script `mutable-dataclass` + `missing-slots` rules |
|
||||
| No null surprise | basedpyright strict `None` analysis |
|
||||
| Constants are const | basedpyright catches `Final` reassignment |
|
||||
| Unused variables | `reportUnusedVariable = "error"` |
|
||||
|
||||
## Sources
|
||||
|
||||
- basedpyright modes: <https://docs.basedpyright.com/latest/configuration/config-files/#type-check-diagnostics-settings>
|
||||
- basedpyright `"all"` vs `"recommended"`: <https://docs.basedpyright.com/latest/configuration/config-files/#recommended-and-all>
|
||||
- basedpyright better defaults: <https://docs.basedpyright.com/latest/benefits-over-pyright/better-defaults/>
|
||||
- ruff rule selection: <https://docs.astral.sh/ruff/linter/#rule-selection>
|
||||
- ruff ANN101/ANN102 removed: <https://github.com/astral-sh/ruff/pull/14384>
|
||||
- Real-world ALL config: <https://github.com/community-of-python/auto-typing-final/blob/main/pyproject.toml>
|
||||
- PEP 735 dependency-groups: <https://peps.python.org/pep-0735/>
|
||||
@@ -0,0 +1,201 @@
|
||||
# Textual TUI
|
||||
|
||||
Textual builds rich, mouse-aware, scrollable, mobile-style TUIs on top of `rich`. Replaces curses, urwid, blessed.
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
uv add textual
|
||||
uv add --dev textual-dev # textual console + run --dev for hot reload
|
||||
```
|
||||
|
||||
## Minimal app
|
||||
|
||||
```python
|
||||
from textual.app import App, ComposeResult
|
||||
from textual.widgets import Header, Footer, Button, Label
|
||||
from textual.containers import Vertical
|
||||
|
||||
|
||||
class CounterApp(App[None]):
|
||||
"""A trivial counter app."""
|
||||
|
||||
BINDINGS = [("q", "quit", "Quit")]
|
||||
CSS = """
|
||||
#count {
|
||||
height: 3;
|
||||
content-align: center middle;
|
||||
background: $boost;
|
||||
}
|
||||
"""
|
||||
|
||||
count: int = 0
|
||||
|
||||
def compose(self) -> ComposeResult:
|
||||
yield Header()
|
||||
with Vertical():
|
||||
yield Label("0", id="count")
|
||||
yield Button("Increment", id="inc", variant="primary")
|
||||
yield Button("Reset", id="reset", variant="warning")
|
||||
yield Footer()
|
||||
|
||||
def on_button_pressed(self, event: Button.Pressed) -> None:
|
||||
if event.button.id == "inc":
|
||||
self.count += 1
|
||||
elif event.button.id == "reset":
|
||||
self.count = 0
|
||||
self.query_one("#count", Label).update(str(self.count))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
CounterApp().run()
|
||||
```
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
uv run python counter.py
|
||||
```
|
||||
|
||||
For hot reload during development:
|
||||
|
||||
```bash
|
||||
uv run textual run --dev counter.py
|
||||
```
|
||||
|
||||
## Reactive attributes
|
||||
|
||||
Textual's `reactive()` descriptor turns a class attribute into something that watches assignments and re-renders automatically. Replaces the manual `query_one` + `update` dance.
|
||||
|
||||
```python
|
||||
from textual.app import App, ComposeResult
|
||||
from textual.reactive import reactive
|
||||
from textual.widgets import Label
|
||||
|
||||
|
||||
class CountWidget(Label):
|
||||
count: reactive[int] = reactive(0)
|
||||
|
||||
def render(self) -> str:
|
||||
return f"Count: {self.count}"
|
||||
|
||||
|
||||
class CounterApp(App[None]):
|
||||
def compose(self) -> ComposeResult:
|
||||
yield CountWidget()
|
||||
|
||||
def on_key(self, event) -> None:
|
||||
if event.key == "space":
|
||||
self.query_one(CountWidget).count += 1
|
||||
```
|
||||
|
||||
`reactive()` triggers `render()` (or `watch_<attr>` and `validate_<attr>` callbacks if defined). Use `recompose=True` if you need to call `compose()` again on change.
|
||||
|
||||
## Async work — workers
|
||||
|
||||
NEVER block the event loop. For network/disk/CPU work, use `@work` (creates a worker) or `run_worker`.
|
||||
|
||||
```python
|
||||
import httpx
|
||||
from textual.app import App, ComposeResult
|
||||
from textual.widgets import Input, Static
|
||||
from textual.work import work
|
||||
|
||||
|
||||
class FetchApp(App[None]):
|
||||
def compose(self) -> ComposeResult:
|
||||
yield Input(placeholder="URL", id="url")
|
||||
yield Static(id="result")
|
||||
|
||||
@work(exclusive=True)
|
||||
async def fetch(self, url: str) -> None:
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
response = await client.get(url)
|
||||
self.query_one("#result", Static).update(f"{response.status_code} - {len(response.text)} bytes")
|
||||
|
||||
def on_input_submitted(self, event: Input.Submitted) -> None:
|
||||
self.fetch(event.value)
|
||||
```
|
||||
|
||||
`exclusive=True` cancels the previous worker if the user submits a new URL before the first finishes. Workers integrate with Textual's lifecycle - they're cancelled when the app exits.
|
||||
|
||||
`@work` is asyncio-flavoured under the hood. That is fine - it does not violate the no-asyncio rule because you are calling Textual's API, not importing asyncio yourself. Inside the worker body, use `httpx.AsyncClient` and other anyio-friendly libraries.
|
||||
|
||||
## Action handlers
|
||||
|
||||
Bind keys to method calls via `BINDINGS` and `action_*` methods.
|
||||
|
||||
```python
|
||||
class App(App):
|
||||
BINDINGS = [
|
||||
("ctrl+s", "save", "Save"),
|
||||
("ctrl+r", "reload", "Reload"),
|
||||
]
|
||||
|
||||
def action_save(self) -> None:
|
||||
# Called on ctrl+s
|
||||
...
|
||||
|
||||
def action_reload(self) -> None:
|
||||
...
|
||||
```
|
||||
|
||||
Bindings can also include the `priority=True` flag to fire before children get a chance.
|
||||
|
||||
## CSS
|
||||
|
||||
Textual's CSS supports selectors, variables (`$primary`, `$boost`), animations. Inline via `CSS = "..."` or external via `CSS_PATH = "app.tcss"`.
|
||||
|
||||
```css
|
||||
Screen {
|
||||
background: $surface;
|
||||
color: $text;
|
||||
layout: vertical;
|
||||
}
|
||||
|
||||
#sidebar {
|
||||
width: 30;
|
||||
background: $boost;
|
||||
}
|
||||
|
||||
Button.danger {
|
||||
background: $error;
|
||||
}
|
||||
```
|
||||
|
||||
Reload with `r` in dev mode (`textual run --dev`).
|
||||
|
||||
## Testing
|
||||
|
||||
```python
|
||||
import pytest
|
||||
from myapp import CounterApp
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_counter_increments() -> None:
|
||||
app = CounterApp()
|
||||
async with app.run_test() as pilot:
|
||||
await pilot.click("#inc")
|
||||
await pilot.click("#inc")
|
||||
assert app.count == 2
|
||||
```
|
||||
|
||||
`pilot.click(selector)`, `pilot.press("q")`, `pilot.pause()` for waiting on the next frame.
|
||||
|
||||
## When NOT to use Textual
|
||||
|
||||
| Need | Use |
|
||||
|---|---|
|
||||
| One-off CLI with structured output | typer + rich |
|
||||
| Progress bar in a script | rich.progress |
|
||||
| Tabular display of query results | rich.table |
|
||||
| Full-screen app with state, input, mouse | Textual |
|
||||
|
||||
A pretty CLI is not a TUI. Reach for Textual when the user expects to navigate a UI, not when you want colours.
|
||||
|
||||
## Sources
|
||||
|
||||
- Textual docs: <https://textual.textualize.io>
|
||||
- Textual tutorial: <https://textual.textualize.io/tutorial/>
|
||||
- API reference: <https://textual.textualize.io/api/>
|
||||
@@ -0,0 +1,176 @@
|
||||
# Type Patterns
|
||||
|
||||
How to use Python's type system to catch bugs at check time, not runtime.
|
||||
|
||||
---
|
||||
|
||||
## NewType — distinct primitives
|
||||
|
||||
Same runtime type, different meaning. The type checker prevents mixing.
|
||||
|
||||
```python
|
||||
from typing import NewType
|
||||
|
||||
UserId = NewType("UserId", int)
|
||||
MovieId = NewType("MovieId", int)
|
||||
Email = NewType("Email", str)
|
||||
Seconds = NewType("Seconds", float)
|
||||
Milliseconds = NewType("Milliseconds", float)
|
||||
|
||||
def get_user(user_id: UserId) -> User: ...
|
||||
def get_movie(movie_id: MovieId) -> Movie: ...
|
||||
def sleep(duration: Seconds) -> None: ...
|
||||
|
||||
uid = UserId(42)
|
||||
mid = MovieId(42)
|
||||
|
||||
get_user(uid) # OK
|
||||
get_user(mid) # type error: MovieId is not UserId
|
||||
get_user(42) # type error: int is not UserId
|
||||
sleep(Milliseconds(100.0)) # type error
|
||||
```
|
||||
|
||||
**Use when**: IDs, indices, keys, units of measurement — any pair where swapping is a bug.
|
||||
**Skip when**: ephemeral local math where branding adds noise with zero safety gain.
|
||||
|
||||
---
|
||||
|
||||
## Final — constants are const
|
||||
|
||||
Module-level constants declare their intent. Reassignment is a type error.
|
||||
|
||||
```python
|
||||
from typing import Final
|
||||
|
||||
MAX_RETRIES: Final = 3
|
||||
API_BASE_URL: Final = "https://api.example.com"
|
||||
DEFAULT_TIMEOUT: Final = 30.0
|
||||
|
||||
MAX_RETRIES = 5 # type error: cannot assign to Final
|
||||
```
|
||||
|
||||
If it changes at runtime, it's not a constant — make it a function parameter or config field.
|
||||
|
||||
---
|
||||
|
||||
## TypeAlias — name complex types
|
||||
|
||||
If a union or generic appears more than once, give it a name.
|
||||
|
||||
```python
|
||||
# Python 3.12+
|
||||
type JsonValue = str | int | float | bool | None | list["JsonValue"] | dict[str, "JsonValue"]
|
||||
type Headers = dict[str, str]
|
||||
type Middleware = Callable[[Request], Awaitable[Response]]
|
||||
|
||||
# Pre-3.12
|
||||
from typing import TypeAlias
|
||||
|
||||
JsonValue: TypeAlias = str | int | float | bool | None | list["JsonValue"] | dict[str, "JsonValue"]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## StrEnum / IntEnum — closed sets
|
||||
|
||||
Any fixed set of known values. No string literals scattered through code.
|
||||
|
||||
```python
|
||||
from enum import StrEnum, IntEnum, unique
|
||||
|
||||
@unique
|
||||
class Role(StrEnum):
|
||||
ADMIN = "admin"
|
||||
USER = "user"
|
||||
GUEST = "guest"
|
||||
|
||||
@unique
|
||||
class HttpStatus(IntEnum):
|
||||
OK = 200
|
||||
NOT_FOUND = 404
|
||||
INTERNAL_ERROR = 500
|
||||
|
||||
# BAD
|
||||
def check_role(role: str) -> bool: ...
|
||||
|
||||
# GOOD
|
||||
def check_role(role: Role) -> bool: ...
|
||||
```
|
||||
|
||||
`StrEnum` when values serialize as strings (API, DB). `IntEnum` for numeric codes. Plain `Enum` for pure labels.
|
||||
|
||||
---
|
||||
|
||||
## Type narrowing — let the checker follow your logic
|
||||
|
||||
`isinstance`, `is None`, and `match` narrow types automatically. Use them instead of `cast`.
|
||||
|
||||
```python
|
||||
def process(value: str | int | None) -> str:
|
||||
if value is None:
|
||||
return "nothing"
|
||||
# checker knows: str | int
|
||||
|
||||
if isinstance(value, str):
|
||||
return value.upper()
|
||||
# checker knows: int
|
||||
|
||||
return str(value * 2)
|
||||
```
|
||||
|
||||
### TypeGuard for custom narrowing
|
||||
|
||||
```python
|
||||
from typing import TypeGuard
|
||||
|
||||
def is_valid_email(value: str) -> TypeGuard[Email]:
|
||||
return "@" in value and "." in value.split("@")[1]
|
||||
|
||||
def send(addr: str) -> None:
|
||||
if not is_valid_email(addr):
|
||||
raise ValueError(addr)
|
||||
# checker knows: addr is Email
|
||||
deliver(addr)
|
||||
```
|
||||
|
||||
### TypeIs (Python 3.13+) — the strict version
|
||||
|
||||
`TypeIs` is stricter than `TypeGuard` — it narrows in both `if` and `else` branches.
|
||||
|
||||
```python
|
||||
from typing import TypeIs
|
||||
|
||||
def is_str(value: str | int) -> TypeIs[str]:
|
||||
return isinstance(value, str)
|
||||
|
||||
def handle(v: str | int) -> None:
|
||||
if is_str(v):
|
||||
print(v.upper()) # checker knows: str
|
||||
else:
|
||||
print(v + 1) # checker knows: int
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Union syntax
|
||||
|
||||
Always `X | Y`. Never `Union[X, Y]` or `Optional[X]`.
|
||||
|
||||
```python
|
||||
# BAD
|
||||
from typing import Union, Optional
|
||||
def f(x: Optional[int]) -> Union[str, int]: ...
|
||||
|
||||
# GOOD
|
||||
def f(x: int | None) -> str | int: ...
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- Python docs: [typing — NewType](https://docs.python.org/3/library/typing.html#newtype)
|
||||
- Python docs: [typing — Final](https://docs.python.org/3/library/typing.html#typing.Final)
|
||||
- Python docs: [typing — TypeGuard](https://docs.python.org/3/library/typing.html#typing.TypeGuard)
|
||||
- PEP 604: [Union syntax X | Y](https://peps.python.org/pep-0604/)
|
||||
- PEP 742: [TypeIs](https://peps.python.org/pep-0742/)
|
||||
@@ -0,0 +1,289 @@
|
||||
|
||||
# Rust Undefined Behavior Exorcist
|
||||
|
||||
You are a UB hunter. Your job is to find, classify, prove, and eliminate every instance of undefined behavior in Rust code. **Miri is your primary weapon** — everything else supplements where Miri cannot reach.
|
||||
|
||||
## Core Philosophy
|
||||
|
||||
1. **Miri first, always.** Before reading a single line of `unsafe`, run Miri. Before proposing a fix, run Miri. After applying a fix, run Miri. Miri is the oracle.
|
||||
2. **Classify before fixing.** Every UB finding gets classified against the 14-category taxonomy (see [ub-taxonomy.md](ub-taxonomy.md)). This prevents misdiagnosis and ensures the fix targets the root cause, not a symptom.
|
||||
3. **Prove the fix.** A fix is not done until Miri passes with full paranoia flags. If Miri cannot run the test (FFI, I/O), the fix is not done until the appropriate sanitizer passes.
|
||||
4. **Bead handoff.** Each resolved UB instance is a "bead" — a discrete, documented, proven fix. Hand it off with: the UB category, the root cause, the fix, and the Miri proof.
|
||||
|
||||
## The UB Taxonomy
|
||||
|
||||
14 categories. The full reference is in [ub-taxonomy.md](ub-taxonomy.md). Memorize the categories; classify every finding:
|
||||
|
||||
| # | Category | Miri? |
|
||||
|---|----------|-------|
|
||||
| 1 | Aliasing violations (Stacked/Tree Borrows) | YES |
|
||||
| 2 | Data races | YES |
|
||||
| 3 | Use-after-free / dangling pointers | YES |
|
||||
| 4 | Uninitialized memory | YES |
|
||||
| 5 | Invalid values (type invariant violations) | YES |
|
||||
| 6 | Misaligned pointer access | YES |
|
||||
| 7 | Pin invariant violations | PARTIAL |
|
||||
| 8 | FFI boundary UB | LIMITED |
|
||||
| 9 | Incorrect Send/Sync implementations | YES (via race) |
|
||||
| 10 | Out-of-bounds memory access | YES |
|
||||
| 11 | Provenance violations | YES (strict mode) |
|
||||
| 12 | Double free / invalid free | YES |
|
||||
| 13 | Library / unsafe contract violations | PARTIAL |
|
||||
| 14 | Unwinding across extern "C" | PARTIAL |
|
||||
|
||||
## The Hunt Workflow
|
||||
|
||||
### Phase 1: Reconnaissance
|
||||
|
||||
1. **Find all `unsafe` blocks and `unsafe impl`s:**
|
||||
```bash
|
||||
rg 'unsafe\s*(fn|impl|{|\{)' --type rust -n
|
||||
```
|
||||
|
||||
2. **Find all `unsafe` trait implementations:**
|
||||
```bash
|
||||
rg 'unsafe\s+impl\s+(Send|Sync)' --type rust -n
|
||||
```
|
||||
|
||||
3. **Find transmute / pointer casts / raw pointer derefs:**
|
||||
```bash
|
||||
rg '(transmute|transmute_copy|from_raw|into_raw|as_ptr|as_mut_ptr|offset|add|sub|read|write|copy|ptr::null)' --type rust -n
|
||||
```
|
||||
|
||||
4. **Find FFI boundaries:**
|
||||
```bash
|
||||
rg 'extern\s+"C"' --type rust -n
|
||||
```
|
||||
|
||||
5. **Count and catalog.** Create a hit list: file, line, `unsafe` category, initial risk assessment (high/medium/low based on the UB taxonomy).
|
||||
|
||||
### Phase 2: Miri Sweep (THE CRITICAL PHASE)
|
||||
|
||||
Run Miri with escalating strictness. **Do not skip any level.**
|
||||
|
||||
**Level 1 — Default (Stacked Borrows):**
|
||||
```bash
|
||||
cargo +nightly miri test 2>&1
|
||||
```
|
||||
|
||||
**Level 2 — Strict Provenance + Symbolic Alignment:**
|
||||
```bash
|
||||
MIRIFLAGS="-Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-backtrace=full" \
|
||||
cargo +nightly miri test 2>&1
|
||||
```
|
||||
|
||||
**Level 3 — Full Paranoia (the audit standard):**
|
||||
```bash
|
||||
MIRIFLAGS="\
|
||||
-Zmiri-strict-provenance \
|
||||
-Zmiri-symbolic-alignment-check \
|
||||
-Zmiri-preemption-rate=0.1 \
|
||||
-Zmiri-backtrace=full \
|
||||
-Zmiri-disable-isolation" \
|
||||
cargo +nightly miri test 2>&1
|
||||
```
|
||||
|
||||
**Level 4 — Tree Borrows (second model confirmation):**
|
||||
```bash
|
||||
MIRIFLAGS="\
|
||||
-Zmiri-tree-borrows \
|
||||
-Zmiri-strict-provenance \
|
||||
-Zmiri-symbolic-alignment-check \
|
||||
-Zmiri-preemption-rate=0.1 \
|
||||
-Zmiri-backtrace=full \
|
||||
-Zmiri-disable-isolation" \
|
||||
cargo +nightly miri test 2>&1
|
||||
```
|
||||
|
||||
**Interpret results:**
|
||||
- Fails at Level 1 → Definite UB. Fix immediately.
|
||||
- Passes Level 1, fails Level 2 → Provenance or alignment UB. Fix.
|
||||
- Passes Levels 1-3, fails Level 4 → Tree Borrows found something Stacked Borrows missed (unusual). Investigate — may be a Tree Borrows false positive, but usually indicates fragile aliasing.
|
||||
- Passes all 4 → Miri-clean. Proceed to supplementary tools.
|
||||
|
||||
### Phase 3: Supplementary Scans
|
||||
|
||||
For code Miri cannot fully cover:
|
||||
|
||||
**Concurrent code with custom atomics:**
|
||||
```bash
|
||||
RUSTFLAGS="--cfg loom" cargo test --lib --release -- loom_tests 2>&1
|
||||
```
|
||||
|
||||
**FFI-heavy code:**
|
||||
```bash
|
||||
RUSTFLAGS="-Zsanitizer=address" cargo +nightly test -Zbuild-std --target $(rustc -vV | rg host | awk '{print $2}') 2>&1
|
||||
```
|
||||
|
||||
**Untrusted input parsing:**
|
||||
```bash
|
||||
cargo +nightly fuzz run <target> -- -max_total_time=300 2>&1
|
||||
```
|
||||
|
||||
### Phase 4: Fix-and-Prove Loop
|
||||
|
||||
For each UB finding:
|
||||
|
||||
1. **Classify** against the 14-category taxonomy.
|
||||
2. **Write the SAFETY comment** explaining what is wrong and what the fix must achieve.
|
||||
3. **Apply the minimal fix.** Do not refactor — fix the UB and nothing else.
|
||||
4. **Run Miri (Level 3 minimum) on the specific test that triggered the UB.**
|
||||
5. **Run Miri (Level 3) on the full test suite** to check for regressions.
|
||||
6. **Document the bead:**
|
||||
```
|
||||
BEAD: [Category #] [Short description]
|
||||
FILE: [path:line]
|
||||
ROOT CAUSE: [one sentence]
|
||||
FIX: [one sentence]
|
||||
PROOF: Miri Level [N] pass — [command used]
|
||||
```
|
||||
|
||||
### Phase 5: Hardening (Post-Fix)
|
||||
|
||||
After all beads are resolved:
|
||||
|
||||
1. **Add Miri to CI** if not already present (see [miri-sanitizers-loom.md](miri-sanitizers-loom.md) for the GitHub Actions config).
|
||||
2. **Add `#[cfg(miri)]` regression tests** for each bead — these are the tests that originally caught the UB, locked in so it never returns.
|
||||
3. **Review SAFETY comments** on every remaining `unsafe` block. Each must name the specific invariant from the taxonomy.
|
||||
4. **Run the full paranoia sweep one final time** to confirm clean.
|
||||
|
||||
## Miri-First Decision Protocol
|
||||
|
||||
When the agent encounters `unsafe` code during ANY Rust task (not just audits):
|
||||
|
||||
```
|
||||
Is there unsafe code in the changeset?
|
||||
YES → Run Miri Level 1 before proceeding.
|
||||
│ Miri fails?
|
||||
│ YES → Stop. Classify. Fix. Prove. Then continue.
|
||||
│ NO → Run Miri Level 2 (strict provenance).
|
||||
│ Miri fails?
|
||||
│ YES → Stop. Classify. Fix. Prove. Then continue.
|
||||
│ NO → Proceed with the original task.
|
||||
NO → Proceed normally.
|
||||
```
|
||||
|
||||
This is not optional. **Every `unsafe` block gets Miri'd before it ships.**
|
||||
|
||||
## SAFETY Comment Standard
|
||||
|
||||
Every `unsafe` block requires a SAFETY comment within 5 lines above it. The comment must:
|
||||
|
||||
1. **Name the UB category** it could trigger (from the taxonomy).
|
||||
2. **State the invariant** that makes this safe.
|
||||
3. **Name who/what guarantees** the invariant (caller contract, type system, runtime check).
|
||||
|
||||
```rust
|
||||
// SAFETY: [Category 4 — Uninitialized Memory]
|
||||
// All N elements have been written to via `ptr::write` in the loop above.
|
||||
// The loop runs exactly `len` times, and `len` was validated against the
|
||||
// allocation size at line 42. MaybeUninit::assume_init is therefore sound.
|
||||
unsafe { buf.assume_init() }
|
||||
```
|
||||
|
||||
Bad SAFETY comments that must be rejected:
|
||||
- `// SAFETY: we know this is safe` — Says nothing.
|
||||
- `// SAFETY: this is fine because we tested it` — Testing does not prove absence of UB.
|
||||
- `// SAFETY: the caller ensures correctness` — Which invariant? What is the contract?
|
||||
- No SAFETY comment at all — Immediate failure.
|
||||
|
||||
## Audit Report Format
|
||||
|
||||
When completing a UB audit, produce a summary:
|
||||
|
||||
```markdown
|
||||
## UB Audit Report
|
||||
|
||||
**Scope:** [crate/module/file]
|
||||
**Miri version:** [output of `cargo +nightly miri --version`]
|
||||
**Date:** [date]
|
||||
|
||||
### Findings
|
||||
|
||||
| # | Category | File:Line | Severity | Status |
|
||||
|---|----------|-----------|----------|--------|
|
||||
| 1 | Aliasing | src/buf.rs:42 | High | Fixed (Bead #1) |
|
||||
| 2 | Uninit | src/ffi.rs:98 | High | Fixed (Bead #2) |
|
||||
|
||||
### Beads
|
||||
|
||||
#### Bead #1: Aliasing violation in buffer resize
|
||||
- **Root cause:** `&mut` created while `&` to same slice existed
|
||||
- **Fix:** Restructured to drop shared ref before taking mutable
|
||||
- **Proof:** `cargo +nightly miri test -- test_buffer_resize` passes Level 3
|
||||
|
||||
### Miri CI Status
|
||||
- [ ] Miri added to CI (Level 2 minimum)
|
||||
- [ ] All SAFETY comments reviewed
|
||||
- [ ] Regression tests added for each bead
|
||||
```
|
||||
|
||||
## Common Fix Patterns
|
||||
|
||||
### Aliasing → Use `UnsafeCell` or restructure borrows
|
||||
```rust
|
||||
// BEFORE (UB: &mut while & exists)
|
||||
let ptr = slice.as_ptr();
|
||||
let mut_ref = &mut slice[0]; // UB: ptr still usable
|
||||
|
||||
// AFTER
|
||||
let mut_ref = &mut slice[0];
|
||||
// ptr is never created / used across the mutable borrow
|
||||
```
|
||||
|
||||
### Uninitialized → Use `MaybeUninit::write` + `assume_init`
|
||||
```rust
|
||||
// BEFORE (UB: mem::uninitialized)
|
||||
let x: T = unsafe { std::mem::uninitialized() };
|
||||
|
||||
// AFTER
|
||||
let x: T = unsafe {
|
||||
let mut uninit = MaybeUninit::<T>::uninit();
|
||||
uninit.write(initial_value);
|
||||
uninit.assume_init()
|
||||
};
|
||||
```
|
||||
|
||||
### Provenance → Use `expose_provenance` / `with_exposed_provenance`
|
||||
```rust
|
||||
// BEFORE (UB: provenance lost)
|
||||
let addr = ptr as usize;
|
||||
let recovered = addr as *const T;
|
||||
|
||||
// AFTER
|
||||
let addr = ptr.expose_provenance();
|
||||
let recovered = std::ptr::with_exposed_provenance::<T>(addr);
|
||||
```
|
||||
|
||||
### Send/Sync → Remove manual impl, use PhantomData
|
||||
```rust
|
||||
// BEFORE (unsound)
|
||||
unsafe impl Send for MyType {}
|
||||
|
||||
// AFTER — if MyType truly needs Send, prove it:
|
||||
// SAFETY: [Category 9 — Send/Sync]
|
||||
// MyType's only non-Send field is `*mut Buffer`. Access to the buffer
|
||||
// is guarded by `self.lock: Mutex<()>`, which provides the
|
||||
// happens-before guarantee required by Send.
|
||||
unsafe impl Send for MyType {}
|
||||
```
|
||||
|
||||
### FFI → Validate at boundary
|
||||
```rust
|
||||
// BEFORE (UB: null pointer from C becomes &T)
|
||||
let result = unsafe { ffi_call() };
|
||||
|
||||
// AFTER
|
||||
let raw = unsafe { ffi_call() };
|
||||
let result = NonNull::new(raw).ok_or(Error::NullFromFfi)?;
|
||||
```
|
||||
|
||||
## Activation
|
||||
|
||||
This skill activates when:
|
||||
- The user requests a "UB audit", "miri sweep", "unsafe audit", "soundness check", "rustonomicon audit", "race hunt"
|
||||
- The agent encounters `unsafe` code during a Rust task and needs to verify it
|
||||
- Miri reports a failure and the agent needs to classify and fix it
|
||||
- The user asks "is this sound?" about Rust code
|
||||
|
||||
**Miri is not optional. Miri is the proof. Ship nothing `unsafe` without Miri's blessing.**
|
||||
+411
@@ -0,0 +1,411 @@
|
||||
# Miri, Sanitizers, Loom, and Fuzzing — The UB Detection Arsenal
|
||||
|
||||
Miri is the **primary weapon**. Everything else is supplementary for the gaps Miri cannot reach.
|
||||
|
||||
---
|
||||
|
||||
## Miri — The First and Last Line of Defense
|
||||
|
||||
### What Miri Is
|
||||
|
||||
Miri is an interpreter for Rust's MIR (Mid-level IR). It executes your test suite inside a virtual machine that tracks every byte of memory for validity, provenance, alignment, initialization, and aliasing. It is **deterministic** — same inputs, same result — and it can find UB that no amount of testing on real hardware will ever trigger.
|
||||
|
||||
### Why Miri Is Non-Negotiable
|
||||
|
||||
- Detects 12 of 14 UB categories (see `ub-taxonomy.md`).
|
||||
- Catches aliasing violations that compile and run correctly on every platform today but are UB that future compiler optimizations will exploit.
|
||||
- Catches data races under a configurable scheduling model.
|
||||
- Catches provenance violations that are impossible to observe on real hardware.
|
||||
- **Zero false positives** — if Miri says it is UB, it is UB. Period.
|
||||
|
||||
### Installation
|
||||
|
||||
```bash
|
||||
rustup install nightly
|
||||
rustup component add miri rust-src --toolchain nightly
|
||||
```
|
||||
|
||||
Verify:
|
||||
```bash
|
||||
cargo +nightly miri --version
|
||||
```
|
||||
|
||||
### Running Miri
|
||||
|
||||
**Default run (Stacked Borrows, standard checks):**
|
||||
```bash
|
||||
cargo +nightly miri test
|
||||
```
|
||||
|
||||
**With nextest (recommended for projects already using nextest):**
|
||||
```bash
|
||||
cargo +nightly miri nextest run
|
||||
```
|
||||
|
||||
**Specific test:**
|
||||
```bash
|
||||
cargo +nightly miri test -- test_name
|
||||
```
|
||||
|
||||
**Run a binary:**
|
||||
```bash
|
||||
cargo +nightly miri run
|
||||
```
|
||||
|
||||
### MIRIFLAGS — The Dial-Up Knobs
|
||||
|
||||
These flags are set via the `MIRIFLAGS` environment variable. The agent should use ALL of the strictness flags during a UB audit.
|
||||
|
||||
#### Aliasing Model
|
||||
|
||||
```bash
|
||||
# Default: Stacked Borrows (strict)
|
||||
cargo +nightly miri test
|
||||
|
||||
# Tree Borrows (newer, more permissive — use as a second pass)
|
||||
MIRIFLAGS="-Zmiri-tree-borrows" cargo +nightly miri test
|
||||
```
|
||||
|
||||
**Protocol:** Run Stacked Borrows first. If it fails, fix it. Then run Tree Borrows to confirm. Code that passes Stacked Borrows is sound under both models.
|
||||
|
||||
#### Strict Provenance
|
||||
|
||||
```bash
|
||||
MIRIFLAGS="-Zmiri-strict-provenance" cargo +nightly miri test
|
||||
```
|
||||
|
||||
Catches `ptr as usize as *const T` roundtrips where provenance is lost. **Should be ON for every audit.**
|
||||
|
||||
#### Symbolic Alignment Checks
|
||||
|
||||
```bash
|
||||
MIRIFLAGS="-Zmiri-symbolic-alignment-check" cargo +nightly miri test
|
||||
```
|
||||
|
||||
Catches alignment UB that happens to be aligned on your machine but is not guaranteed by the type system.
|
||||
|
||||
#### Data Race Detection Tuning
|
||||
|
||||
```bash
|
||||
# Increase preemption rate to stress-test race conditions
|
||||
MIRIFLAGS="-Zmiri-preemption-rate=0.5" cargo +nightly miri test
|
||||
|
||||
# Disable preemption (sequential scheduling — fewer races found but deterministic)
|
||||
MIRIFLAGS="-Zmiri-preemption-rate=0" cargo +nightly miri test
|
||||
```
|
||||
|
||||
#### The Full Paranoia Sweep (Use This for Audits)
|
||||
|
||||
```bash
|
||||
MIRIFLAGS="\
|
||||
-Zmiri-strict-provenance \
|
||||
-Zmiri-symbolic-alignment-check \
|
||||
-Zmiri-preemption-rate=0.1 \
|
||||
-Zmiri-backtrace=full \
|
||||
-Zmiri-disable-isolation" \
|
||||
cargo +nightly miri test
|
||||
```
|
||||
|
||||
Then a second pass with Tree Borrows:
|
||||
```bash
|
||||
MIRIFLAGS="\
|
||||
-Zmiri-tree-borrows \
|
||||
-Zmiri-strict-provenance \
|
||||
-Zmiri-symbolic-alignment-check \
|
||||
-Zmiri-preemption-rate=0.1 \
|
||||
-Zmiri-backtrace=full \
|
||||
-Zmiri-disable-isolation" \
|
||||
cargo +nightly miri test
|
||||
```
|
||||
|
||||
#### Isolation and I/O
|
||||
|
||||
Miri runs in isolation by default — no file I/O, no network, no system calls. If your tests need the filesystem:
|
||||
```bash
|
||||
MIRIFLAGS="-Zmiri-disable-isolation" cargo +nightly miri test
|
||||
```
|
||||
|
||||
Use sparingly — isolation is a feature, not a limitation. Tests that need I/O should have a separate `#[cfg(not(miri))]` path.
|
||||
|
||||
### Miri Limitations
|
||||
|
||||
| Cannot do | Workaround |
|
||||
|-----------|-----------|
|
||||
| Execute FFI / C code | ASAN, MSAN, Valgrind |
|
||||
| Run I/O-heavy tests (default) | `-Zmiri-disable-isolation` or `#[cfg(not(miri))]` |
|
||||
| Exhaustive interleaving exploration | loom |
|
||||
| Find performance bugs | criterion, flamegraph |
|
||||
| Run inline assembly | skip with `#[cfg(not(miri))]` |
|
||||
| Test OS-specific behavior | real hardware + sanitizers |
|
||||
|
||||
### Miri in CI
|
||||
|
||||
```yaml
|
||||
# GitHub Actions example
|
||||
miri:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
with:
|
||||
components: miri, rust-src
|
||||
- name: Miri test (Stacked Borrows + strict provenance)
|
||||
run: |
|
||||
MIRIFLAGS="-Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-backtrace=full" \
|
||||
cargo +nightly miri test
|
||||
- name: Miri test (Tree Borrows)
|
||||
run: |
|
||||
MIRIFLAGS="-Zmiri-tree-borrows -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-backtrace=full" \
|
||||
cargo +nightly miri test
|
||||
```
|
||||
|
||||
### Miri-Incompatible Test Gating
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
#[cfg_attr(miri, ignore)] // Miri cannot run this (FFI, I/O, inline asm)
|
||||
fn test_requires_real_hardware() {
|
||||
// ...
|
||||
}
|
||||
|
||||
// Or conditionally compile the test body:
|
||||
#[test]
|
||||
fn test_with_miri_fallback() {
|
||||
#[cfg(miri)]
|
||||
{
|
||||
// Simplified version that avoids FFI
|
||||
}
|
||||
#[cfg(not(miri))]
|
||||
{
|
||||
// Full version with FFI
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Sanitizers — Where Miri Cannot Reach
|
||||
|
||||
Sanitizers are compiler instrumentation passes. They run your actual binary on real hardware with extra checks injected. Use them for FFI, I/O-heavy code, and integration tests.
|
||||
|
||||
### AddressSanitizer (ASAN)
|
||||
|
||||
Detects: use-after-free, buffer overflow, stack-use-after-return, double-free, memory leaks.
|
||||
|
||||
```bash
|
||||
RUSTFLAGS="-Zsanitizer=address" cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu
|
||||
```
|
||||
|
||||
On macOS:
|
||||
```bash
|
||||
RUSTFLAGS="-Zsanitizer=address" cargo +nightly test -Zbuild-std --target aarch64-apple-darwin
|
||||
```
|
||||
|
||||
### ThreadSanitizer (TSAN)
|
||||
|
||||
Detects: data races on non-atomic accesses across threads.
|
||||
|
||||
```bash
|
||||
RUSTFLAGS="-Zsanitizer=thread" cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu
|
||||
```
|
||||
|
||||
**When to use over Miri:** Integration tests involving real threads + real I/O + FFI. Miri's data-race detector is superior for pure-Rust code.
|
||||
|
||||
### MemorySanitizer (MSAN)
|
||||
|
||||
Detects: reads of uninitialized memory.
|
||||
|
||||
```bash
|
||||
RUSTFLAGS="-Zsanitizer=memory -Zsanitizer-memory-track-origins" cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu
|
||||
```
|
||||
|
||||
**When to use over Miri:** FFI code where C/C++ may return uninitialized memory into Rust.
|
||||
|
||||
### UndefinedBehaviorSanitizer (UBSAN)
|
||||
|
||||
Detects: integer overflow, misaligned access, null dereference, and other C/C++-style UB at the LLVM level.
|
||||
|
||||
```bash
|
||||
RUSTFLAGS="-Zsanitizer=undefined" cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu
|
||||
```
|
||||
|
||||
### Sanitizer Limitations
|
||||
|
||||
- Require nightly + `-Zbuild-std` (rebuilds the standard library with instrumentation).
|
||||
- MSAN requires ALL dependencies (including C libs) to be instrumented — practically hard.
|
||||
- Cannot catch aliasing violations (that is Miri's domain).
|
||||
- Significant runtime overhead (2-15x slower).
|
||||
- Linux has the best support; macOS works for ASAN; Windows support is minimal.
|
||||
|
||||
---
|
||||
|
||||
## Loom — Exhaustive Concurrency Testing
|
||||
|
||||
Loom explores all possible thread interleavings of a bounded concurrent program. It is mandatory for lock-free and wait-free primitives.
|
||||
|
||||
### When to Use Loom
|
||||
|
||||
- Any `unsafe` code involving atomics with ordering weaker than `SeqCst`.
|
||||
- Custom lock implementations.
|
||||
- Lock-free queues, stacks, or other concurrent data structures.
|
||||
- Any code where you chose `Relaxed`, `Acquire`, or `Release` ordering.
|
||||
|
||||
### When NOT to Use Loom
|
||||
|
||||
- Code using only `Mutex`/`RwLock` from std or `parking_lot` — the locks are sound, your usage is the question, and Miri + TSAN cover that.
|
||||
- Async code (loom does not model async runtimes — use `tokio::test` + Miri instead).
|
||||
|
||||
### Setup
|
||||
|
||||
```toml
|
||||
[dev-dependencies]
|
||||
loom = "0.7"
|
||||
```
|
||||
|
||||
### Loom Test Pattern
|
||||
|
||||
```rust
|
||||
#[cfg(loom)]
|
||||
mod loom_tests {
|
||||
use loom::sync::atomic::{AtomicUsize, Ordering};
|
||||
use loom::sync::Arc;
|
||||
use loom::thread;
|
||||
|
||||
#[test]
|
||||
fn concurrent_increment_is_sound() {
|
||||
loom::model(|| {
|
||||
let counter = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let threads: Vec<_> = (0..2).map(|_| {
|
||||
let c = counter.clone();
|
||||
thread::spawn(move || {
|
||||
c.fetch_add(1, Ordering::SeqCst);
|
||||
})
|
||||
}).collect();
|
||||
|
||||
for t in threads {
|
||||
t.join().unwrap();
|
||||
}
|
||||
|
||||
assert_eq!(counter.load(Ordering::SeqCst), 2);
|
||||
});
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Conditional Compilation for Loom
|
||||
|
||||
```rust
|
||||
#[cfg(loom)]
|
||||
use loom::sync::atomic::{AtomicUsize, Ordering};
|
||||
#[cfg(not(loom))]
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
```
|
||||
|
||||
### Running Loom Tests
|
||||
|
||||
```bash
|
||||
# Loom tests only (use cfg flag)
|
||||
RUSTFLAGS="--cfg loom" cargo test --lib -- loom_tests
|
||||
|
||||
# With release optimizations (loom is slow)
|
||||
RUSTFLAGS="--cfg loom" cargo test --lib --release -- loom_tests
|
||||
```
|
||||
|
||||
### Loom + Miri Interaction
|
||||
|
||||
Loom and Miri solve different problems:
|
||||
- **Miri** checks a single execution for UB (aliasing, validity, provenance).
|
||||
- **Loom** checks all interleavings for correctness (ordering, atomicity).
|
||||
|
||||
Run BOTH on lock-free code:
|
||||
```bash
|
||||
# Step 1: loom for interleaving correctness
|
||||
RUSTFLAGS="--cfg loom" cargo test --lib --release -- loom_tests
|
||||
|
||||
# Step 2: Miri for UB in each path
|
||||
cargo +nightly miri test -- concurrent_tests
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Cargo-Fuzz — Property-Based UB Hunting
|
||||
|
||||
Fuzzing generates random inputs to maximize code coverage and find crashes, panics, and UB.
|
||||
|
||||
### Setup
|
||||
|
||||
```bash
|
||||
cargo install cargo-fuzz
|
||||
cargo fuzz init
|
||||
```
|
||||
|
||||
### Fuzz Target
|
||||
|
||||
```rust
|
||||
// fuzz/fuzz_targets/parse_input.rs
|
||||
#![no_main]
|
||||
use libfuzzer_sys::fuzz_target;
|
||||
|
||||
fuzz_target!(|data: &[u8]| {
|
||||
// Your parsing/deserialization/processing code here.
|
||||
// If it panics or triggers UB, the fuzzer catches it.
|
||||
let _ = my_crate::parse(data);
|
||||
});
|
||||
```
|
||||
|
||||
### Running
|
||||
|
||||
```bash
|
||||
# Run until interrupted
|
||||
cargo +nightly fuzz run parse_input
|
||||
|
||||
# Run with ASAN (catches memory bugs in unsafe code)
|
||||
cargo +nightly fuzz run parse_input -- -rss_limit_mb=4096
|
||||
|
||||
# Minimize a crashing input
|
||||
cargo +nightly fuzz tmin parse_input artifacts/parse_input/crash-xxxxx
|
||||
```
|
||||
|
||||
### Fuzz + Miri Pipeline
|
||||
|
||||
When the fuzzer finds a crashing input:
|
||||
1. Minimize it with `cargo fuzz tmin`.
|
||||
2. Add it as a regression test.
|
||||
3. Run the regression test under Miri to classify whether it is a panic (safe) or UB (must fix).
|
||||
|
||||
```bash
|
||||
# After adding the input as a test case:
|
||||
cargo +nightly miri test -- test_fuzz_regression_001
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Tool Selection Decision Tree
|
||||
|
||||
```
|
||||
Start
|
||||
│
|
||||
├── Is it pure Rust (no FFI, no I/O)?
|
||||
│ YES → Miri (full paranoia flags)
|
||||
│ │ └── Also: loom (if atomics/lock-free)
|
||||
│ │ └── Also: proptest (if parsing/serialization)
|
||||
│ │ └── Also: cargo-fuzz (if untrusted input)
|
||||
│ │
|
||||
│ NO → Does it involve FFI?
|
||||
│ YES → ASAN + MSAN on integration tests
|
||||
│ │ └── Miri on the Rust-side handling
|
||||
│ │ └── cbindgen in CI for layout verification
|
||||
│ │
|
||||
│ NO → Is it I/O-heavy?
|
||||
│ YES → TSAN for thread safety
|
||||
│ │ └── Miri with -Zmiri-disable-isolation where possible
|
||||
│ │
|
||||
│ NO → Miri (full paranoia flags)
|
||||
│
|
||||
└── Always: Miri is the default. Other tools supplement.
|
||||
```
|
||||
|
||||
## The One Rule
|
||||
|
||||
> **When in doubt, run Miri.** If Miri cannot run it, write a version it can run, and test that under Miri. Then test the real version under sanitizers. Never ship `unsafe` code that has not passed Miri.
|
||||
@@ -0,0 +1,269 @@
|
||||
# Rust Undefined Behavior Taxonomy
|
||||
|
||||
Every category of UB the Rust compiler, Miri, and the language specification recognize. The agent must know the full surface to hunt systematically. Each entry names the UB class, its root cause, canonical trigger, Miri detection status, and the canonical fix.
|
||||
|
||||
## 1. Aliasing Violations (Stacked Borrows / Tree Borrows)
|
||||
|
||||
**Root cause:** Two pointers access the same memory in ways that violate Rust's borrowing model — even through raw pointers inside `unsafe`.
|
||||
|
||||
**Canonical triggers:**
|
||||
- Creating a `&mut T` while another `&T` or `&mut T` to the same location exists.
|
||||
- Dereferencing a raw pointer derived from a reference after that reference was invalidated (e.g., `&mut` was retaken).
|
||||
- Calling `slice::from_raw_parts_mut` on overlapping regions.
|
||||
- Interior mutability through `UnsafeCell` without going through the `UnsafeCell` API.
|
||||
- Casting `&T` to `*mut T` and writing through it (even via FFI).
|
||||
|
||||
**Miri detection:** YES — Stacked Borrows is the default model. Tree Borrows (`-Zmiri-tree-borrows`) is the newer, more permissive model. Run both:
|
||||
```bash
|
||||
cargo +nightly miri test # Stacked Borrows (stricter)
|
||||
MIRIFLAGS="-Zmiri-tree-borrows" cargo +nightly miri test # Tree Borrows (relaxed)
|
||||
```
|
||||
If code passes Tree Borrows but fails Stacked Borrows, it is *likely* sound but *possibly* relying on unspecified behavior. Fix it anyway — Stacked Borrows is the conservative bet.
|
||||
|
||||
**Fix pattern:** Use `UnsafeCell` for all interior mutability. Never cast `&T` to `*mut T`. Derive mutable pointers from `*mut T` obtained via `UnsafeCell::get()` or `addr_of_mut!()`.
|
||||
|
||||
---
|
||||
|
||||
## 2. Data Races
|
||||
|
||||
**Root cause:** Two threads access the same non-atomic memory location, at least one is a write, and there is no happens-before ordering between them.
|
||||
|
||||
**Canonical triggers:**
|
||||
- `unsafe impl Send for T` on a type containing `*mut U` without synchronization.
|
||||
- `unsafe impl Sync for T` on a type containing `Cell<T>` or `UnsafeCell<T>` without a lock.
|
||||
- Using `std::ptr::write` from multiple threads to the same allocation.
|
||||
- Shared `&T` where `T` has interior mutability but no atomic/lock guard.
|
||||
|
||||
**Miri detection:** YES — Miri's data-race detector is on by default. It detects races on non-atomic accesses. For **preemptive scheduling** stress, use:
|
||||
```bash
|
||||
MIRIFLAGS="-Zmiri-preemption-rate=0.1" cargo +nightly miri test
|
||||
```
|
||||
|
||||
**Complementary tools:** `loom` for exhaustive interleaving exploration on lock-free algorithms. ThreadSanitizer (TSAN) for integration tests Miri cannot run (I/O, FFI).
|
||||
|
||||
**Fix pattern:** Wrap in `Mutex`/`RwLock`/`AtomicXxx`. Never `unsafe impl Sync` unless you can name the synchronization primitive guarding every mutable field.
|
||||
|
||||
---
|
||||
|
||||
## 3. Use After Free / Dangling Pointers
|
||||
|
||||
**Root cause:** A pointer or reference outlives the allocation it points to.
|
||||
|
||||
**Canonical triggers:**
|
||||
- Returning a reference to a local variable (compiler catches most, but raw pointers escape).
|
||||
- `Box::into_raw` → manual `Box::from_raw` with wrong lifetime.
|
||||
- `Vec` reallocation invalidating raw pointers obtained from `as_ptr()` / `as_mut_ptr()`.
|
||||
- `Pin<Box<T>>` unpinned and moved after self-referential pointers were set up.
|
||||
|
||||
**Miri detection:** YES — allocation tracking catches use-after-free on the exact operation.
|
||||
|
||||
**Fix pattern:** Borrow checker for references. For raw pointers: tie pointer validity to an explicit lifetime via a `PhantomData<&'a T>` in the wrapper, or use arena allocation (`bumpalo`) so all pointers share one lifetime.
|
||||
|
||||
---
|
||||
|
||||
## 4. Uninitialized Memory
|
||||
|
||||
**Root cause:** Reading a value from memory that was never written to.
|
||||
|
||||
**Canonical triggers:**
|
||||
- `MaybeUninit::assume_init()` before all bytes are written.
|
||||
- `mem::uninitialized()` (deprecated, still compiles).
|
||||
- `alloc::alloc(layout)` returns uninitialized memory — reading it before writing is UB.
|
||||
- Padding bytes in structs read via `transmute` or raw pointer casts.
|
||||
- `read_unaligned` on uninitialized memory.
|
||||
|
||||
**Miri detection:** YES — tracks initialization state per byte. Catches partial-init structs, padding reads, and premature `assume_init`.
|
||||
|
||||
**Fix pattern:** Use `MaybeUninit::zeroed()` when zero-init is acceptable. Write every field before calling `assume_init()`. Use `MaybeUninit::write()` instead of raw pointer writes. Never `transmute` structs with padding unless you zeroed the padding.
|
||||
|
||||
---
|
||||
|
||||
## 5. Invalid Values (Type Invariant Violations)
|
||||
|
||||
**Root cause:** Producing a value that violates the type's validity invariant.
|
||||
|
||||
**Canonical triggers:**
|
||||
- `bool` not 0 or 1.
|
||||
- `char` outside Unicode scalar range.
|
||||
- Enum discriminant not matching any variant.
|
||||
- `NonZeroU32` containing 0.
|
||||
- `&T` or `&mut T` that is null or dangling.
|
||||
- `str` containing non-UTF-8 bytes.
|
||||
- `fn` pointer that is null.
|
||||
|
||||
**Miri detection:** YES — validity checks are on by default. Extra strictness:
|
||||
```bash
|
||||
MIRIFLAGS="-Zmiri-strict-provenance" cargo +nightly miri test
|
||||
```
|
||||
|
||||
**Fix pattern:** Validate before transmuting. Use `TryFrom` at boundaries. Never `transmute` to enum types — use a checked conversion function.
|
||||
|
||||
---
|
||||
|
||||
## 6. Misaligned Pointer Access
|
||||
|
||||
**Root cause:** Dereferencing a pointer that is not aligned to the type's required alignment.
|
||||
|
||||
**Canonical triggers:**
|
||||
- Casting `*const u8` to `*const u64` and dereferencing (alignment goes from 1 to 8).
|
||||
- `#[repr(packed)]` struct field references (the compiler warns, but raw pointers bypass the warning).
|
||||
- Network buffer parsing where offsets are arbitrary.
|
||||
|
||||
**Miri detection:** YES — immediate trap on misaligned read/write.
|
||||
|
||||
**Fix pattern:** Use `read_unaligned` / `write_unaligned` for packed data. Use `bytemuck` or `zerocopy` for safe reinterpretation with alignment checks.
|
||||
|
||||
---
|
||||
|
||||
## 7. Violating `Pin` Invariants
|
||||
|
||||
**Root cause:** Moving a value that was pinned and relied on its address stability (self-referential types, intrusive linked lists).
|
||||
|
||||
**Canonical triggers:**
|
||||
- `mem::swap` on a `Pin<&mut T>` after `unsafe` deref.
|
||||
- Implementing `Unpin` for a type that contains self-referential pointers.
|
||||
- Manually calling `Pin::new_unchecked` on a movable allocation.
|
||||
|
||||
**Miri detection:** PARTIAL — Miri detects the resulting aliasing/use-after-free if the self-referential pointer is actually used. It does not detect "Pin contract violated but pointer was never dereferenced."
|
||||
|
||||
**Fix pattern:** Never `impl Unpin` for self-referential types. Use `pin_project` or `pin_project_lite` for safe pin projections. Review every `Pin::new_unchecked` call.
|
||||
|
||||
---
|
||||
|
||||
## 8. FFI Boundary UB
|
||||
|
||||
**Root cause:** Mismatch between Rust's ABI expectations and the foreign code's actual behavior.
|
||||
|
||||
**Canonical triggers:**
|
||||
- C function returning uninitialized memory into a Rust `&T`.
|
||||
- Wrong `#[repr(C)]` layout (padding differs between platforms).
|
||||
- Passing a Rust `enum` to C without `#[repr(C)]` or `#[repr(i32)]`.
|
||||
- Null pointer passed where C expects non-null (and Rust wraps it in `&T`).
|
||||
- C code writing to Rust-owned memory through a pointer Rust considers immutable.
|
||||
- Forgetting to mark FFI functions as `unsafe extern "C"`.
|
||||
- longjmp/setjmp across Rust frames (unwinding UB).
|
||||
|
||||
**Miri detection:** LIMITED — Miri cannot execute foreign code. It detects UB in the Rust-side handling of FFI return values.
|
||||
|
||||
**Complementary tools:** AddressSanitizer (ASAN), MemorySanitizer (MSAN) for detecting actual FFI-side corruption. Valgrind as a last resort.
|
||||
|
||||
**Fix pattern:** Validate every FFI return at the boundary. Use `Option<NonNull<T>>` for nullable pointers. Use `CStr`/`CString` for strings. Add `cbindgen` to CI to verify layout agreement. Wrap every FFI call in a safe Rust function that checks preconditions.
|
||||
|
||||
---
|
||||
|
||||
## 9. Incorrect `Send` / `Sync` Implementations
|
||||
|
||||
**Root cause:** Manually implementing `Send` or `Sync` for a type that does not actually uphold the required invariant.
|
||||
|
||||
**Canonical triggers:**
|
||||
- `unsafe impl Send for Wrapper(*mut T)` when `T` is not `Send`.
|
||||
- `unsafe impl Sync for Wrapper(UnsafeCell<T>)` without a lock, atomic, or other synchronization.
|
||||
- Types containing `Rc<T>` with a manual `Send` impl (Rc is explicitly !Send).
|
||||
|
||||
**Miri detection:** YES for the *resulting* data race if exercised. Miri's data-race detector will fire when two threads access the same location unsynchronized.
|
||||
|
||||
**Fix pattern:** Never manually implement `Send`/`Sync` unless you can write a SAFETY proof naming the synchronization mechanism. Use `PhantomData<*const ()>` to opt-out of auto-`Send`/`Sync` when in doubt.
|
||||
|
||||
---
|
||||
|
||||
## 10. Out-of-Bounds Memory Access
|
||||
|
||||
**Root cause:** Pointer arithmetic or indexing that escapes the allocation.
|
||||
|
||||
**Canonical triggers:**
|
||||
- `ptr.offset(n)` where `n` exceeds the allocation size.
|
||||
- `slice::from_raw_parts(ptr, len)` where `len` is too large.
|
||||
- Off-by-one in manual buffer management.
|
||||
- Integer overflow in size calculations leading to undersized allocation.
|
||||
|
||||
**Miri detection:** YES — allocation-precise bounds checking.
|
||||
|
||||
**Fix pattern:** Use checked arithmetic (`checked_add`, `checked_mul`) for size calculations. Use `slice::from_raw_parts` only with validated lengths. Prefer safe indexing (`get()`, iterators) over raw pointer arithmetic.
|
||||
|
||||
---
|
||||
|
||||
## 11. Provenance Violations
|
||||
|
||||
**Root cause:** Using a pointer whose provenance does not grant access to the target memory, even if the address is numerically correct.
|
||||
|
||||
**Canonical triggers:**
|
||||
- Casting an integer to a pointer and dereferencing it (`addr as *const T`).
|
||||
- Roundtripping a pointer through `usize` and back (`ptr as usize as *const T`) — the provenance is lost.
|
||||
- Using `ptr::from_exposed_addr` without a corresponding `ptr.expose_provenance()`.
|
||||
|
||||
**Miri detection:** YES with strict provenance:
|
||||
```bash
|
||||
MIRIFLAGS="-Zmiri-strict-provenance" cargo +nightly miri test
|
||||
```
|
||||
|
||||
**Fix pattern:** Use `ptr::with_exposed_provenance` / `ptr.expose_provenance()` for legitimate int-to-ptr roundtrips. Avoid `as usize as *const T` entirely. Use `sptr` crate for provenance-safe pointer manipulation on stable.
|
||||
|
||||
---
|
||||
|
||||
## 12. Double Free / Invalid Free
|
||||
|
||||
**Root cause:** Freeing the same allocation twice, or freeing memory not obtained from the allocator.
|
||||
|
||||
**Canonical triggers:**
|
||||
- `Box::from_raw` called twice on the same pointer.
|
||||
- Manual `dealloc` on a pointer already freed.
|
||||
- `ManuallyDrop` dropped explicitly then the outer type also drops it.
|
||||
|
||||
**Miri detection:** YES — immediate trap.
|
||||
|
||||
**Fix pattern:** Enforce single ownership via RAII. Use `ManuallyDrop` with extreme care — document who is responsible for the drop. Never clone a raw pointer and `Box::from_raw` both copies.
|
||||
|
||||
---
|
||||
|
||||
## 13. Library / Unsafe Contract Violations
|
||||
|
||||
**Root cause:** Violating the documented safety invariant of a safe or unsafe API, where the library author relied on the invariant for soundness.
|
||||
|
||||
**Canonical triggers:**
|
||||
- `Vec::set_len(n)` where the first `n` elements are not initialized.
|
||||
- `String::from_utf8_unchecked` on non-UTF-8 bytes.
|
||||
- `HashMap` key mutated after insertion (violates hash invariant — not UB per se, but unsound and Miri may detect downstream effects).
|
||||
- `BTreeMap` key with broken `Ord` impl (the standard library assumes a total order).
|
||||
|
||||
**Miri detection:** DEPENDS — Miri catches the downstream UB (e.g., reading uninitialized bytes from a `Vec` with inflated len). It does not catch "you violated the documented contract" if no memory-level UB results.
|
||||
|
||||
**Fix pattern:** Read the `# Safety` section of every `unsafe fn` you call. Document the invariant in your SAFETY comment. When in doubt, use the safe API and pay the cost.
|
||||
|
||||
---
|
||||
|
||||
## 14. Unwinding Across `extern "C"` Boundaries
|
||||
|
||||
**Root cause:** A Rust panic unwinding through a frame that uses the C calling convention.
|
||||
|
||||
**Canonical triggers:**
|
||||
- `panic!()` inside a `#[no_mangle] extern "C" fn` callback passed to C code.
|
||||
- `unwrap()` inside FFI callbacks.
|
||||
|
||||
**Miri detection:** PARTIAL — Miri does not model foreign unwinding, but it can detect the immediate UB if the panic reaches the FFI boundary.
|
||||
|
||||
**Fix pattern:** Use `std::panic::catch_unwind` at every FFI entry point. Mark FFI callbacks as `extern "C-unwind"` when panic propagation is intentional (nightly). Prefer returning `Result`-like error codes from FFI callbacks.
|
||||
|
||||
---
|
||||
|
||||
## Summary Table
|
||||
|
||||
| # | Category | Miri Detects? | Complementary Tool |
|
||||
|---|----------|--------------|-------------------|
|
||||
| 1 | Aliasing (Stacked/Tree Borrows) | YES | — |
|
||||
| 2 | Data races | YES | loom, TSAN |
|
||||
| 3 | Use-after-free / dangling | YES | ASAN |
|
||||
| 4 | Uninitialized memory | YES | MSAN |
|
||||
| 5 | Invalid values | YES | — |
|
||||
| 6 | Misaligned access | YES | UBSAN |
|
||||
| 7 | Pin invariant violation | PARTIAL | manual review |
|
||||
| 8 | FFI boundary UB | LIMITED | ASAN, MSAN, Valgrind |
|
||||
| 9 | Incorrect Send/Sync | YES (via race) | loom |
|
||||
| 10 | Out-of-bounds access | YES | ASAN |
|
||||
| 11 | Provenance violations | YES (strict mode) | — |
|
||||
| 12 | Double free | YES | ASAN |
|
||||
| 13 | Library contract violations | PARTIAL | proptest, fuzzing |
|
||||
| 14 | Unwinding across FFI | PARTIAL | — |
|
||||
|
||||
## Miri Coverage Assessment
|
||||
|
||||
Miri catches categories 1-6, 9-12 with high confidence. Categories 7, 8, 13, 14 require supplementary tools or manual audit. **Miri is the single highest-leverage tool** — it should run on every PR that touches `unsafe`, and ideally on the full test suite regularly.
|
||||
@@ -0,0 +1,317 @@
|
||||
# Rust Programmer
|
||||
|
||||
Production Rust in 2026. **Explicit allocation, compile-time proof, zero hidden cost.** Type-state-first, unsafe-banished-by-default, agent-proof.
|
||||
|
||||
## Identity — What Kind of Rust You Write
|
||||
|
||||
You write Rust that looks like a Zig programmer designed it and a Rust compiler enforces it. Every allocation is visible. Every cost is explicit. Every invariant is encoded in the type system. Every cleanup is deterministic. The borrow checker, lifetime analysis, trait bounds, and `miri` then guarantee what Zig leaves to discipline.
|
||||
|
||||
**Five pillars, every file, no exceptions:**
|
||||
|
||||
| Pillar | Default Behavior | Reference |
|
||||
|---|---|---|
|
||||
| **Explicit allocation** | Arena for hot paths, `&[T]`/`Cow` over `Vec`/`String` in signatures, `try_*` when allocation can fail | [zero-cost-safety.md §1](zero-cost-safety.md) |
|
||||
| **Compile-time proof** | `const fn` everything const-eligible, `const { assert!(...) }` for compile-time guards, const generics for sized buffers | [zero-cost-safety.md §2](zero-cost-safety.md) |
|
||||
| **Zero hidden cost** | Slice-based APIs where caller owns memory, no hidden `.clone()`/`.to_string()`, `Cow` to defer allocation | [zero-cost-safety.md §3](zero-cost-safety.md) |
|
||||
| **Type-encoded invariants** | Newtype wrappers for every semantic unit, type-state for state machines, branded IDs | [type-state.md](type-state.md) |
|
||||
| **Deterministic cleanup** | `scopeguard::guard` for errdefer, `Drop` for RAII, defuse-on-success for rollback | [zero-cost-safety.md §5](zero-cost-safety.md) |
|
||||
|
||||
The two highest-leverage tools Rust gives a coding agent:
|
||||
|
||||
1. **Bounded polymorphism** (traits). Real, machine-checked, composable constraints.
|
||||
2. **Newtype-as-coordinate-space.** `Point<Screen>` and `Point<World>` are distinct types — the agent literally cannot pass one where the other is expected. This is the `euclid` crate pattern; generalize ruthlessly to money, durations, IDs, byte offsets, char offsets, paths rooted at different bases. Full patterns → [type-state.md](type-state.md).
|
||||
|
||||
---
|
||||
|
||||
## Hard Rules (Every `.rs` File)
|
||||
|
||||
### 1. No `unwrap()`, No `expect()` Outside Tests
|
||||
|
||||
```rust
|
||||
// WRONG
|
||||
let val = map.get("key").unwrap();
|
||||
|
||||
// RIGHT — propagate or provide context
|
||||
let val = map.get("key").context("missing 'key' in config")?;
|
||||
```
|
||||
|
||||
Typed errors for libraries ([thiserror](https://docs.rs/thiserror)), ad-hoc errors for binaries ([anyhow](https://docs.rs/anyhow) / [color-eyre](https://docs.rs/color-eyre)). Full stack → [libraries.md](libraries.md).
|
||||
|
||||
### 2. No `unsafe` Without Miri Proof
|
||||
|
||||
If `unsafe` is unavoidable, you have miri. Run it. Always. **Load [`../rust-ub/README.md`](../rust-ub/README.md) plus every file under [`../rust-ub/`](../rust-ub/)** for the full UB taxonomy, Miri escalation protocol (4 strictness levels), and the fix-and-prove workflow. Every `unsafe` block needs the three components from [unsafe-discipline.md](unsafe-discipline.md): safe wrapper, `// SAFETY:` comment, miri test.
|
||||
|
||||
```bash
|
||||
cargo +nightly miri nextest run
|
||||
```
|
||||
|
||||
### 3. Explicit Allocation — Arena by Default in Hot Paths
|
||||
|
||||
**Do not scatter `Box::new()` / `Vec::new()` across hot loops.** Use arena allocation to make allocation scope visible and bulk-freeable. Full recipes → [zero-cost-safety.md §1](zero-cost-safety.md).
|
||||
|
||||
```rust
|
||||
use bumpalo::Bump;
|
||||
|
||||
fn parse_frame<'a>(arena: &'a Bump, raw: &[u8]) -> Frame<'a> {
|
||||
let header = arena.alloc(parse_header(raw));
|
||||
let payload = arena.alloc_slice_copy(&raw[HEADER_LEN..]);
|
||||
Frame { header, payload }
|
||||
}
|
||||
// Caller owns arena. Caller decides when memory dies. Zero individual frees.
|
||||
```
|
||||
|
||||
When arena is overkill (simple CLI, one-shot allocation), `Vec`/`String` are fine — but **function signatures still prefer borrows**:
|
||||
|
||||
```rust
|
||||
// WRONG — forces caller to allocate
|
||||
fn process(input: String) -> String { ... }
|
||||
|
||||
// RIGHT — caller chooses allocation strategy
|
||||
fn process(input: &str) -> Cow<'_, str> { ... }
|
||||
|
||||
// BEST for hot paths — zero allocation, caller provides buffer
|
||||
fn process(input: &[u8], output: &mut [u8]) -> usize { ... }
|
||||
```
|
||||
|
||||
### 4. Compile-Time First — const fn Everything Const-Eligible
|
||||
|
||||
If a function CAN be `const fn`, it MUST be `const fn`. Full recipes → [zero-cost-safety.md §2](zero-cost-safety.md).
|
||||
|
||||
```rust
|
||||
// Lookup tables computed at compile time — zero runtime cost
|
||||
const CRC_TABLE: [u32; 256] = {
|
||||
let mut table = [0u32; 256];
|
||||
let mut i = 0;
|
||||
while i < 256 {
|
||||
let mut crc = i as u32;
|
||||
let mut j = 0;
|
||||
while j < 8 {
|
||||
crc = if crc & 1 != 0 { (crc >> 1) ^ 0xEDB88320 } else { crc >> 1 };
|
||||
j += 1;
|
||||
}
|
||||
table[i] = crc;
|
||||
i += 1;
|
||||
}
|
||||
table
|
||||
};
|
||||
|
||||
// Compile-time assertions — catch violations at build time, not runtime
|
||||
const { assert!(std::mem::size_of::<Header>() == 12, "Header must be 12 bytes") };
|
||||
```
|
||||
|
||||
Use `const generics` for stack-allocated buffers with compile-time size:
|
||||
|
||||
```rust
|
||||
struct RingBuffer<T, const N: usize> {
|
||||
data: [MaybeUninit<T>; N],
|
||||
head: usize,
|
||||
len: usize,
|
||||
}
|
||||
```
|
||||
|
||||
### 5. Scope Guards — Deterministic Cleanup on Every Path
|
||||
|
||||
Zig's `errdefer` in Rust. Full recipes → [zero-cost-safety.md §5](zero-cost-safety.md).
|
||||
|
||||
```rust
|
||||
use scopeguard::guard;
|
||||
|
||||
fn deploy(artifact: &Path) -> Result<(), DeployError> {
|
||||
let backup = snapshot_current()?;
|
||||
// errdefer: restore on failure
|
||||
let rollback = guard(backup, |b| { let _ = restore(&b); });
|
||||
|
||||
upload(artifact)?;
|
||||
health_check()?;
|
||||
|
||||
// Success: defuse the guard
|
||||
scopeguard::ScopeGuard::into_inner(rollback);
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
### 6. Bit-Level Layout — zerocopy for Wire Formats
|
||||
|
||||
Never hand-write `transmute` or pointer casts for parsing binary data. Full recipes → [zero-cost-safety.md §4](zero-cost-safety.md).
|
||||
|
||||
```rust
|
||||
use zerocopy::{FromBytes, IntoBytes, KnownLayout, Immutable};
|
||||
|
||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable)]
|
||||
#[repr(C)]
|
||||
struct PacketHeader {
|
||||
magic: [u8; 4],
|
||||
version: u8,
|
||||
flags: u8,
|
||||
length: [u8; 2], // use byte array for packed fields, decode via from_le_bytes
|
||||
}
|
||||
```
|
||||
|
||||
### 7. Exhaustive Match — No Wildcard on Enums You Control
|
||||
|
||||
```rust
|
||||
// WRONG — silently ignores new variants
|
||||
match status {
|
||||
Status::Ok => handle_ok(),
|
||||
_ => handle_error(),
|
||||
}
|
||||
|
||||
// RIGHT — compiler forces update when variants change
|
||||
match status {
|
||||
Status::Ok => handle_ok(),
|
||||
Status::NotFound => handle_not_found(),
|
||||
Status::Timeout => handle_timeout(),
|
||||
}
|
||||
```
|
||||
|
||||
For `#[non_exhaustive]` enums from external crates, the wildcard `_` is required — but add a `tracing::warn!` in the catch-all so you notice when new variants appear.
|
||||
|
||||
### 8. Type-State Over Runtime Checks
|
||||
|
||||
Never `if self.state == State::Validated`. Encode states as distinct types so the compiler refuses invalid transitions. Full patterns → [type-state.md](type-state.md).
|
||||
|
||||
```rust
|
||||
struct Order<S: OrderState> { data: OrderData, _state: PhantomData<S> }
|
||||
struct Draft;
|
||||
struct Validated;
|
||||
struct Paid;
|
||||
|
||||
impl Order<Draft> {
|
||||
fn validate(self) -> Result<Order<Validated>, ValidationError> { ... }
|
||||
}
|
||||
impl Order<Validated> {
|
||||
fn pay(self, payment: Payment) -> Result<Order<Paid>, PaymentError> { ... }
|
||||
}
|
||||
// Order<Draft> has no .pay() method. Compiler enforces the workflow.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Standard Library Defaults
|
||||
|
||||
Full decision tree with rationale and code snippets → [libraries.md](libraries.md).
|
||||
|
||||
| Category | Crate | Why |
|
||||
|---|---|---|
|
||||
| Async runtime | `tokio` | Ecosystem standard. Patterns → [async-tokio.md](async-tokio.md) |
|
||||
| HTTP server | `axum` + `tower` | Type-safe extractors, tower middleware. Stack → [axum-stack.md](axum-stack.md) |
|
||||
| CLI | `clap` derive + `color-eyre` | Typed args, beautiful errors. Stack → [clap-stack.md](clap-stack.md) |
|
||||
| Serialization | `serde` + `serde_json` | Non-negotiable for any boundary type |
|
||||
| Error (library) | `thiserror` | Derive `Error` with zero boilerplate |
|
||||
| Error (binary) | `anyhow` / `color-eyre` | Context-rich ad-hoc errors |
|
||||
| Database | `sqlx` (compile-time checked) | No runtime SQL surprises |
|
||||
| Arena alloc | `bumpalo` / `typed-arena` | Explicit allocation scope. Patterns → [zero-cost-safety.md §1](zero-cost-safety.md) |
|
||||
| Zero-copy parse | `zerocopy` | Safe binary parsing, no transmute. Patterns → [zero-cost-safety.md §4](zero-cost-safety.md) |
|
||||
| Scope guard | `scopeguard` | errdefer/defer. Patterns → [zero-cost-safety.md §5](zero-cost-safety.md) |
|
||||
| Stack collections | `smallvec` / `arrayvec` / `tinyvec` | Stack-first, heap-spillover. Patterns → [zero-cost-safety.md §3](zero-cost-safety.md) |
|
||||
| Bitfield | `bitfield` / `modular-bitfield` | Bit-packed flags. Patterns → [zero-cost-safety.md §4](zero-cost-safety.md) |
|
||||
| Testing | `proptest` + `insta` | Property + snapshot tests. Patterns → [proptest-insta.md](proptest-insta.md) |
|
||||
| Concurrency | `tokio::sync` / `parking_lot` | Channel-first, lock-second. Patterns → [concurrency.md](concurrency.md) |
|
||||
|
||||
---
|
||||
|
||||
## Cargo Strict Configuration
|
||||
|
||||
Every new project gets the strict lint config from [cargo-strict.md](cargo-strict.md). The non-negotiable CI gate:
|
||||
|
||||
```bash
|
||||
cargo fmt --all -- --check && \
|
||||
cargo clippy --all-targets --all-features -- -D warnings && \
|
||||
cargo nextest run && \
|
||||
cargo +nightly miri nextest run # when unsafe is involved
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Code Review Checklist (Post-Write, Every PR)
|
||||
|
||||
Run through this list after writing any Rust code. Every item links to its recipe.
|
||||
|
||||
| # | Check | Fix Reference |
|
||||
|---|---|---|
|
||||
| 1 | Every function signature prefers `&[T]`/`&str`/`Cow` over owned types | [zero-cost-safety.md §3](zero-cost-safety.md) |
|
||||
| 2 | Hot-path allocations use arena (`bumpalo`) not scattered `Box`/`Vec` | [zero-cost-safety.md §1](zero-cost-safety.md) |
|
||||
| 3 | Const-eligible functions are `const fn` | [zero-cost-safety.md §2](zero-cost-safety.md) |
|
||||
| 4 | Lookup tables / config constants computed at compile time | [zero-cost-safety.md §2](zero-cost-safety.md) |
|
||||
| 5 | Binary format parsing uses `zerocopy`, not `transmute` | [zero-cost-safety.md §4](zero-cost-safety.md) |
|
||||
| 6 | Cleanup logic uses `scopeguard` or `Drop`, never manual `if err` cleanup | [zero-cost-safety.md §5](zero-cost-safety.md) |
|
||||
| 7 | Distinct semantic units are newtypes, not primitive aliases | [type-state.md](type-state.md) |
|
||||
| 8 | State machines use type-state, not runtime `if state ==` | [type-state.md](type-state.md) |
|
||||
| 9 | No `unwrap()`/`expect()` outside `#[cfg(test)]` | [libraries.md](libraries.md) |
|
||||
| 10 | Every `unsafe` has SAFETY comment + miri test | [unsafe-discipline.md](unsafe-discipline.md), [../rust-ub/](../rust-ub/) |
|
||||
| 11 | Match on owned enums is exhaustive (no `_ =>`) | This file §7 |
|
||||
| 12 | Clippy pedantic passes with zero warnings | [cargo-strict.md](cargo-strict.md) |
|
||||
| 13 | Property tests exist for any function with a nontrivial domain | [proptest-insta.md](proptest-insta.md) |
|
||||
| 14 | Concurrency uses channels first, locks second, atomics last | [concurrency.md](concurrency.md) |
|
||||
| 15 | Async code uses `JoinSet` for structured concurrency | [async-tokio.md](async-tokio.md) |
|
||||
|
||||
---
|
||||
|
||||
## Default Cargo.toml Dependencies — Zero-Cost Safety Stack
|
||||
|
||||
Every new project starts with these alongside the standard deps from [cargo-strict.md](cargo-strict.md):
|
||||
|
||||
```toml
|
||||
# Zero-cost safety stack
|
||||
bumpalo = { version = "3", features = ["collections"] }
|
||||
scopeguard = "1"
|
||||
smallvec = { version = "1", features = ["union", "const_generics"] }
|
||||
zerocopy = { version = "0.8", features = ["derive"] }
|
||||
|
||||
# Add when needed:
|
||||
# typed-arena = "2" # homogeneous arena
|
||||
# arrayvec = "0.7" # fixed-capacity stack vec
|
||||
# tinyvec = { version = "1", features = ["alloc"] }
|
||||
# bitfield = "0.17" # bit-packed flags
|
||||
# modular-bitfield = "0.11" # richer bitfield API
|
||||
# bytemuck = { version = "1", features = ["derive"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Reference Index
|
||||
|
||||
| File | When to Load |
|
||||
|---|---|
|
||||
| [zero-cost-safety.md](zero-cost-safety.md) | Arena, allocator, const fn, comptime, zero-alloc, bitfield, repr, scopeguard, errdefer, Zig-like patterns |
|
||||
| [type-state.md](type-state.md) | Newtype wrappers, type-state machines, branded IDs, phantom types |
|
||||
| [unsafe-discipline.md](unsafe-discipline.md) | Any `unsafe` block — SAFETY comments, safe wrappers, miri proof |
|
||||
| [libraries.md](libraries.md) | Library selection, crate decision tree, dependency audit |
|
||||
| [cargo-strict.md](cargo-strict.md) | Project bootstrap, lint config, CI gate commands |
|
||||
| [async-tokio.md](async-tokio.md) | Async runtime, spawning, cancellation, `JoinSet`, `select!` |
|
||||
| [axum-stack.md](axum-stack.md) | HTTP services — axum + sqlx + tower + tracing |
|
||||
| [clap-stack.md](clap-stack.md) | CLI tools — clap derive + color-eyre + indicatif |
|
||||
| [concurrency.md](concurrency.md) | Locks, atomics, channels, loom model checker |
|
||||
| [proptest-insta.md](proptest-insta.md) | Property tests, snapshot tests, round-trip invariants |
|
||||
| [one-liners.md](one-liners.md) | `rust-script` one-liners, disposable scripts, inline deps |
|
||||
| [../rust-ub/README.md](../rust-ub/README.md) | UB hunting — miri escalation, sanitizers, fuzzing |
|
||||
| [../rust-ub/ub-taxonomy.md](../rust-ub/ub-taxonomy.md) | 14-category UB taxonomy with detection status |
|
||||
| [../rust-ub/miri-sanitizers-loom.md](../rust-ub/miri-sanitizers-loom.md) | Miri flags, ASAN/TSAN/MSAN, loom, cargo-fuzz |
|
||||
|
||||
---
|
||||
|
||||
## The Shape of Every Function
|
||||
|
||||
```rust
|
||||
/// One-line doc explaining WHAT, not HOW.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns `FooError::Bar` when the input is invalid.
|
||||
const fn frobnicate<'a>(
|
||||
arena: &'a Bump, // explicit allocator when arena is in play
|
||||
input: &[u8], // borrow, not owned
|
||||
output: &mut [u8], // caller-provided buffer
|
||||
) -> Result<&'a Frob, FrobError> {
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
**Why this shape:** the caller sees every cost. Allocation scope is the arena's lifetime. Input is borrowed. Output buffer is caller-owned. Error is typed. The compiler enforces all of it.
|
||||
|
||||
---
|
||||
|
||||
## Activation
|
||||
|
||||
This skill activates whenever you are writing or modifying any `.rs` file or `Cargo.toml`. One-off scripts get the strict treatment too — `rust-script` + the same lints, the same gates. Details → [one-liners.md](one-liners.md).
|
||||
|
||||
**The promise:** production hygiene with throwaway ergonomics. Explicit allocation, compile-time proof, zero hidden cost, and **agent-proof safety at any volume**.
|
||||
@@ -0,0 +1,299 @@
|
||||
# Async with Tokio
|
||||
|
||||
Structured concurrency, cancellation, blocking-work isolation, channel selection. The patterns the agent should reach for by default.
|
||||
|
||||
## Runtime selection
|
||||
|
||||
```rust
|
||||
// Default for services and CLIs that do real work
|
||||
#[tokio::main(flavor = "multi_thread", worker_threads = 8)]
|
||||
async fn main() -> anyhow::Result<()> { ... }
|
||||
|
||||
// For tiny CLIs or wasm where you measured single-thread is enough
|
||||
#[tokio::main(flavor = "current_thread")]
|
||||
async fn main() -> anyhow::Result<()> { ... }
|
||||
```
|
||||
|
||||
Pick worker count explicitly. The default (`num_cpus`) is fine for servers; for desktop tools you usually want 2-4.
|
||||
|
||||
## Spawning
|
||||
|
||||
`tokio::spawn` returns a `JoinHandle<T>`. The future runs to completion even if the handle is dropped (detached). To enforce structured concurrency, use `JoinSet`:
|
||||
|
||||
```rust
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
let mut set = JoinSet::new();
|
||||
for url in urls {
|
||||
let client = client.clone();
|
||||
set.spawn(async move { fetch(&client, &url).await });
|
||||
}
|
||||
|
||||
let mut results = Vec::new();
|
||||
while let Some(joined) = set.join_next().await {
|
||||
match joined {
|
||||
Ok(Ok(body)) => results.push(body),
|
||||
Ok(Err(error)) => tracing::warn!(%error, "fetch failed"),
|
||||
Err(panicked) if panicked.is_panic() => {
|
||||
tracing::error!(?panicked, "worker panicked");
|
||||
// Choose: re-raise, or continue with degraded result set.
|
||||
}
|
||||
Err(other) => tracing::error!(?other, "worker join error"),
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`JoinSet`:
|
||||
- Knows when all spawned tasks finish.
|
||||
- Dropping the set aborts every still-running task.
|
||||
- Lets you handle failures one by one rather than all-or-nothing.
|
||||
|
||||
For wait-for-all semantics with one type, `join!`:
|
||||
|
||||
```rust
|
||||
let (a, b, c) = tokio::join!(load_a(), load_b(), load_c());
|
||||
let a = a?; let b = b?; let c = c?;
|
||||
```
|
||||
|
||||
For first-of-many, `select!`:
|
||||
|
||||
```rust
|
||||
tokio::select! {
|
||||
biased; // bias to top-to-bottom checking when ordering matters
|
||||
_ = shutdown.recv() => {
|
||||
tracing::info!("shutdown signal");
|
||||
return Ok(());
|
||||
}
|
||||
request = listener.accept() => {
|
||||
handle_request(request?).await?;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Without `biased`, branches are polled in random order each iteration (good for fairness). Use `biased` only when you need deterministic priority (shutdown signal first, etc).
|
||||
|
||||
## Cancellation
|
||||
|
||||
A future is cancelled when it is dropped (e.g., the `select!` arm wins another branch). **Always think: if this future is dropped mid-await, what state is left behind?**
|
||||
|
||||
Cancel-safe futures (you can drop without lasting effect):
|
||||
- `recv()` on channels
|
||||
- `accept()` on listeners
|
||||
- `wait_for` on `watch::Receiver`
|
||||
- `read_buf`/`write_all` on streams **only when buffers are owned by the future**, otherwise no
|
||||
|
||||
Cancel-unsafe futures (dropping mid-way leaves partial state):
|
||||
- Manual `read_exact` into an external buffer
|
||||
- Custom futures that perform partial side effects before suspending
|
||||
|
||||
If a function is cancel-unsafe, document it in a rustdoc `# Cancel Safety` section.
|
||||
|
||||
To explicitly opt out of cancellation, use `tokio_util::sync::CancellationToken`:
|
||||
|
||||
```rust
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
let token = CancellationToken::new();
|
||||
let child = token.child_token();
|
||||
tokio::spawn(async move {
|
||||
tokio::select! {
|
||||
_ = child.cancelled() => { /* clean up */ }
|
||||
result = work() => { /* normal */ }
|
||||
}
|
||||
});
|
||||
// later
|
||||
token.cancel();
|
||||
```
|
||||
|
||||
Pass child tokens down the call tree so the whole tree can be cancelled together.
|
||||
|
||||
## Timeouts
|
||||
|
||||
```rust
|
||||
use tokio::time::{timeout, Duration};
|
||||
|
||||
match timeout(Duration::from_secs(5), fetch(url)).await {
|
||||
Ok(Ok(body)) => Ok(body),
|
||||
Ok(Err(error)) => Err(error.into()),
|
||||
Err(_elapsed) => Err(anyhow::anyhow!("timed out fetching {url}")),
|
||||
}
|
||||
```
|
||||
|
||||
Set timeouts on every external I/O boundary. Defaults of "wait forever" are bugs.
|
||||
|
||||
## Blocking work
|
||||
|
||||
NEVER block inside an async task. Symptoms: deadlock, every future stalled, latency cliffs.
|
||||
|
||||
Heavy CPU or sync I/O → `spawn_blocking`:
|
||||
|
||||
```rust
|
||||
let result = tokio::task::spawn_blocking(|| {
|
||||
// CPU-bound: parsing, hashing, image processing
|
||||
// Or sync I/O: rusqlite, OS APIs without async wrappers
|
||||
expensive_pure_computation()
|
||||
}).await?;
|
||||
```
|
||||
|
||||
Long-running blocking jobs (more than ~1 second of CPU) → use a dedicated thread pool (`rayon`), not tokio's blocking pool which is sized for short bursts.
|
||||
|
||||
## Channels
|
||||
|
||||
| Need | Use |
|
||||
|---|---|
|
||||
| 1-many producers → 1 consumer, async | `tokio::sync::mpsc::channel(cap)` |
|
||||
| Same as above, both sync + async | `flume::bounded(cap)` |
|
||||
| 1 → many fan-out, latest-value semantics | `tokio::sync::watch::channel(initial)` |
|
||||
| 1 → many fan-out, queued | `tokio::sync::broadcast::channel(cap)` |
|
||||
| One-shot reply | `tokio::sync::oneshot::channel()` |
|
||||
| Backpressure-driven stream of items | `tokio::sync::mpsc::Receiver` + `ReceiverStream` |
|
||||
|
||||
Mpsc pattern:
|
||||
|
||||
```rust
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel::<Job>(256);
|
||||
|
||||
tokio::spawn(async move {
|
||||
while let Some(job) = rx.recv().await {
|
||||
if let Err(error) = process(job).await {
|
||||
tracing::warn!(%error, "job failed");
|
||||
}
|
||||
}
|
||||
tracing::info!("queue closed, shutting down worker");
|
||||
});
|
||||
|
||||
tx.send(Job { ... }).await?; // blocks if full, applies backpressure
|
||||
```
|
||||
|
||||
Always bound channels. Unbounded channels are a memory leak waiting to happen.
|
||||
|
||||
## Streams
|
||||
|
||||
`futures::Stream` is the async analogue of `Iterator`. Use it for paginated fetches, long-poll responses, file lines.
|
||||
|
||||
```rust
|
||||
use futures::stream::{StreamExt, TryStreamExt};
|
||||
|
||||
let urls: Vec<String> = ...;
|
||||
let bodies: Vec<String> = futures::stream::iter(urls)
|
||||
.map(|url| async move { fetch(&url).await })
|
||||
.buffer_unordered(8) // up to 8 in flight
|
||||
.try_collect()
|
||||
.await?;
|
||||
```
|
||||
|
||||
`buffer_unordered(n)` is the throttle. Use it instead of spawning N tasks manually.
|
||||
|
||||
For producing a stream from a channel:
|
||||
|
||||
```rust
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
|
||||
let (tx, rx) = tokio::sync::mpsc::channel::<Event>(64);
|
||||
let stream = ReceiverStream::new(rx);
|
||||
serve_sse(stream).await
|
||||
```
|
||||
|
||||
## Graceful shutdown
|
||||
|
||||
```rust
|
||||
use tokio::signal;
|
||||
|
||||
async fn shutdown_signal() {
|
||||
let ctrl_c = async { signal::ctrl_c().await.expect("ctrl_c handler") };
|
||||
#[cfg(unix)]
|
||||
let terminate = async {
|
||||
signal::unix::signal(signal::unix::SignalKind::terminate())
|
||||
.expect("install signal handler")
|
||||
.recv()
|
||||
.await;
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let terminate = std::future::pending::<()>();
|
||||
|
||||
tokio::select! {
|
||||
_ = ctrl_c => {},
|
||||
_ = terminate => {},
|
||||
}
|
||||
tracing::info!("shutdown signal received");
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let token = CancellationToken::new();
|
||||
let server = tokio::spawn(run_server(token.child_token()));
|
||||
shutdown_signal().await;
|
||||
token.cancel();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), server).await;
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
Pattern: catch signal → cancel a token shared with the server → server's `select!` arms see the cancel and exit cleanly → wait with a timeout so a hung worker can't deadlock shutdown.
|
||||
|
||||
## Concurrency primitives
|
||||
|
||||
- `tokio::sync::Mutex` — async mutex. Use for state shared between async tasks. **Do not hold across `.await` without thinking** (you'll serialize the whole system).
|
||||
- `tokio::sync::RwLock` — async read-write lock. Same caveat.
|
||||
- `parking_lot::Mutex` — sync mutex, faster than `std::sync::Mutex`, no poisoning. Use when the lock is held briefly and you do not need to `.await` while holding it.
|
||||
- `tokio::sync::Semaphore` — bound concurrent operations. Perfect for "max 10 in-flight HTTP requests" or "max 3 DB writers".
|
||||
|
||||
```rust
|
||||
let sem = Arc::new(tokio::sync::Semaphore::new(10));
|
||||
for url in urls {
|
||||
let permit = sem.clone().acquire_owned().await?;
|
||||
tokio::spawn(async move {
|
||||
let _permit = permit; // released on task end
|
||||
fetch(&url).await
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
## Common mistakes
|
||||
|
||||
1. **Holding a sync mutex across `.await`.** Compiles and runs, deadlocks at scale. Solution: refactor to release before await, or use `tokio::sync::Mutex`.
|
||||
2. **Forgetting `?` on `JoinHandle`.** A panicked task returns `Err(JoinError)`; if you `.await` and ignore, panics are silently swallowed.
|
||||
3. **`tokio::spawn` instead of `JoinSet`.** Detached tasks survive past their parent, causing leaks. Default to `JoinSet` for structured concurrency.
|
||||
4. **Unbounded channels.** Always set a capacity.
|
||||
5. **`block_on` inside an async context.** Causes deadlock under `current_thread` runtime, performance cliff under `multi_thread`.
|
||||
6. **CPU-heavy work in async fn.** Move to `spawn_blocking` or `rayon`.
|
||||
7. **No timeout on external I/O.** Every `await` that touches the network or filesystem needs `tokio::time::timeout` wrapping.
|
||||
|
||||
## Testing async code
|
||||
|
||||
```rust
|
||||
#[tokio::test]
|
||||
async fn fetches_and_parses() {
|
||||
let server = wiremock::MockServer::start().await;
|
||||
wiremock::Mock::given(wiremock::matchers::method("GET"))
|
||||
.respond_with(wiremock::ResponseTemplate::new(200).set_body_string("{\"id\":1}"))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let result = my_client::fetch(&server.uri()).await.unwrap();
|
||||
assert_eq!(result.id, 1);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn parallel_work() { ... }
|
||||
```
|
||||
|
||||
For time-sensitive tests, advance virtual time:
|
||||
|
||||
```rust
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn time_travel() {
|
||||
let start = tokio::time::Instant::now();
|
||||
tokio::time::sleep(Duration::from_secs(3600)).await;
|
||||
assert!(start.elapsed() >= Duration::from_secs(3600));
|
||||
// Real wallclock elapsed: ~0ms.
|
||||
}
|
||||
```
|
||||
|
||||
## When NOT to use async
|
||||
|
||||
- Single-threaded CPU-heavy code that does no I/O — plain `fn` + `rayon` is simpler and often faster.
|
||||
- Trivial scripts that do one HTTP call — `ureq` (sync) is simpler.
|
||||
- FFI heavy code where the FFI side is sync.
|
||||
|
||||
Async pays off when you have many concurrent I/O operations or need cancellation as a first-class primitive.
|
||||
@@ -0,0 +1,467 @@
|
||||
# axum + sqlx + tracing + tower — HTTP API Stack
|
||||
|
||||
The canonical production HTTP service in Rust 2026.
|
||||
|
||||
## Cargo.toml dependencies
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
axum = { version = "0.8", features = ["macros", "tracing", "ws", "multipart"] }
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tower = "0.5"
|
||||
tower-http = { version = "0.6", features = [
|
||||
"trace", "compression-gzip", "compression-br",
|
||||
"timeout", "cors", "request-id", "sensitive-headers",
|
||||
"limit", "set-header",
|
||||
] }
|
||||
|
||||
# Errors / observability
|
||||
anyhow = "1"
|
||||
thiserror = "2"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
|
||||
color-eyre = "0.6"
|
||||
|
||||
# Database
|
||||
sqlx = { version = "0.8", features = [
|
||||
"runtime-tokio-rustls", "postgres", "uuid", "macros",
|
||||
"migrate", "json",
|
||||
] }
|
||||
|
||||
# Serialization / validation
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
validator = { version = "0.18", features = ["derive"] }
|
||||
|
||||
# Types
|
||||
uuid = { version = "1", features = ["v4", "v7", "serde"] }
|
||||
jiff = { version = "0.1", features = ["serde"] }
|
||||
|
||||
# Config
|
||||
config = { version = "0.14", features = ["toml", "yaml"] }
|
||||
secrecy = { version = "0.10", features = ["serde"] }
|
||||
|
||||
# OpenAPI (optional but recommended)
|
||||
utoipa = { version = "5", features = ["axum_extras", "uuid", "chrono"] }
|
||||
utoipa-axum = "0.1"
|
||||
utoipa-swagger-ui = { version = "8", features = ["axum"] }
|
||||
```
|
||||
|
||||
## Project structure
|
||||
|
||||
```
|
||||
src/
|
||||
main.rs # binary entry
|
||||
lib.rs # re-exports + app builder
|
||||
config.rs # Settings type + loader
|
||||
state.rs # AppState (shared via Arc)
|
||||
routes/
|
||||
mod.rs # Router::new() composition
|
||||
health.rs
|
||||
users.rs
|
||||
middleware/
|
||||
mod.rs
|
||||
auth.rs
|
||||
request_id.rs
|
||||
models/
|
||||
mod.rs
|
||||
user.rs
|
||||
error.rs # AppError + IntoResponse impl
|
||||
db/
|
||||
mod.rs
|
||||
migrations/
|
||||
migrations/ # sqlx migrations
|
||||
```
|
||||
|
||||
## Error type
|
||||
|
||||
```rust
|
||||
// src/error.rs
|
||||
use axum::{http::StatusCode, response::{IntoResponse, Response}, Json};
|
||||
use serde_json::json;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum AppError {
|
||||
#[error("not found")]
|
||||
NotFound,
|
||||
#[error("unauthorized")]
|
||||
Unauthorized,
|
||||
#[error("validation: {0}")]
|
||||
Validation(String),
|
||||
#[error("conflict: {0}")]
|
||||
Conflict(String),
|
||||
#[error("internal")]
|
||||
Internal(#[from] anyhow::Error),
|
||||
#[error("database")]
|
||||
Database(#[from] sqlx::Error),
|
||||
}
|
||||
|
||||
impl IntoResponse for AppError {
|
||||
fn into_response(self) -> Response {
|
||||
let (status, code, message) = match &self {
|
||||
AppError::NotFound => (StatusCode::NOT_FOUND, "not_found", self.to_string()),
|
||||
AppError::Unauthorized => (StatusCode::UNAUTHORIZED, "unauthorized", "unauthorized".into()),
|
||||
AppError::Validation(m) => (StatusCode::UNPROCESSABLE_ENTITY, "validation", m.clone()),
|
||||
AppError::Conflict(m) => (StatusCode::CONFLICT, "conflict", m.clone()),
|
||||
AppError::Database(e) => {
|
||||
tracing::error!(error = ?e, "database error");
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, "database", "internal".into())
|
||||
}
|
||||
AppError::Internal(e) => {
|
||||
tracing::error!(error = ?e, "internal error");
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, "internal", "internal".into())
|
||||
}
|
||||
};
|
||||
(status, Json(json!({"error": {"code": code, "message": message}}))).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
pub type AppResult<T> = std::result::Result<T, AppError>;
|
||||
```
|
||||
|
||||
Pattern: business errors return `AppResult<T>`; the `IntoResponse` impl translates them to HTTP. `sqlx::Error` and `anyhow::Error` auto-convert via `From`. Internal-bucket errors are logged but never leak their `Debug` representation to clients.
|
||||
|
||||
## AppState
|
||||
|
||||
```rust
|
||||
// src/state.rs
|
||||
use std::sync::Arc;
|
||||
use sqlx::PgPool;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AppState {
|
||||
pub db: PgPool,
|
||||
pub config: Arc<crate::config::Settings>,
|
||||
pub http: reqwest::Client,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
pub async fn new(config: crate::config::Settings) -> anyhow::Result<Self> {
|
||||
let db = sqlx::postgres::PgPoolOptions::new()
|
||||
.max_connections(config.db.max_connections)
|
||||
.acquire_timeout(std::time::Duration::from_secs(3))
|
||||
.connect(config.db.url.expose_secret())
|
||||
.await?;
|
||||
sqlx::migrate!("./migrations").run(&db).await?;
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(15))
|
||||
.user_agent(concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")))
|
||||
.build()?;
|
||||
Ok(Self { db, config: Arc::new(config), http })
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Route handler
|
||||
|
||||
```rust
|
||||
// src/routes/users.rs
|
||||
use axum::{
|
||||
extract::{Path, State},
|
||||
Json,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
use validator::Validate;
|
||||
|
||||
use crate::{error::{AppError, AppResult}, state::AppState};
|
||||
|
||||
#[derive(Debug, Deserialize, Validate)]
|
||||
pub struct CreateUser {
|
||||
#[validate(email)]
|
||||
pub email: String,
|
||||
#[validate(length(min = 1, max = 100))]
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, sqlx::FromRow)]
|
||||
pub struct User {
|
||||
pub id: Uuid,
|
||||
pub email: String,
|
||||
pub name: String,
|
||||
pub created_at: jiff::Timestamp,
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(state, body))]
|
||||
pub async fn create_user(
|
||||
State(state): State<AppState>,
|
||||
Json(body): Json<CreateUser>,
|
||||
) -> AppResult<(axum::http::StatusCode, Json<User>)> {
|
||||
body.validate().map_err(|e| AppError::Validation(e.to_string()))?;
|
||||
|
||||
let id = Uuid::now_v7();
|
||||
let user = sqlx::query_as!(
|
||||
User,
|
||||
r#"INSERT INTO users (id, email, name, created_at)
|
||||
VALUES ($1, $2, $3, NOW())
|
||||
RETURNING id, email, name, created_at as "created_at: jiff::Timestamp""#,
|
||||
id, body.email, body.name
|
||||
)
|
||||
.fetch_one(&state.db)
|
||||
.await
|
||||
.map_err(|e| match &e {
|
||||
sqlx::Error::Database(db) if db.code().as_deref() == Some("23505") =>
|
||||
AppError::Conflict("email already exists".into()),
|
||||
_ => AppError::Database(e),
|
||||
})?;
|
||||
|
||||
Ok((axum::http::StatusCode::CREATED, Json(user)))
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(state))]
|
||||
pub async fn get_user(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<Uuid>,
|
||||
) -> AppResult<Json<User>> {
|
||||
sqlx::query_as!(
|
||||
User,
|
||||
r#"SELECT id, email, name, created_at as "created_at: jiff::Timestamp"
|
||||
FROM users WHERE id = $1"#,
|
||||
id
|
||||
)
|
||||
.fetch_optional(&state.db)
|
||||
.await?
|
||||
.map(Json)
|
||||
.ok_or(AppError::NotFound)
|
||||
}
|
||||
```
|
||||
|
||||
## Router assembly
|
||||
|
||||
```rust
|
||||
// src/routes/mod.rs
|
||||
use axum::{routing::{get, post}, Router};
|
||||
use tower_http::{
|
||||
compression::CompressionLayer,
|
||||
cors::CorsLayer,
|
||||
request_id::{MakeRequestUuid, PropagateRequestIdLayer, SetRequestIdLayer},
|
||||
sensitive_headers::SetSensitiveHeadersLayer,
|
||||
timeout::TimeoutLayer,
|
||||
trace::{DefaultMakeSpan, DefaultOnResponse, TraceLayer},
|
||||
};
|
||||
use std::time::Duration;
|
||||
use crate::state::AppState;
|
||||
|
||||
mod health;
|
||||
mod users;
|
||||
|
||||
pub fn router(state: AppState) -> Router {
|
||||
let api = Router::new()
|
||||
.route("/health", get(health::handler))
|
||||
.route("/users", post(users::create_user))
|
||||
.route("/users/:id", get(users::get_user))
|
||||
.with_state(state);
|
||||
|
||||
Router::new()
|
||||
.nest("/api/v1", api)
|
||||
.layer(
|
||||
tower::ServiceBuilder::new()
|
||||
.layer(SetSensitiveHeadersLayer::new([
|
||||
axum::http::header::AUTHORIZATION,
|
||||
axum::http::header::COOKIE,
|
||||
]))
|
||||
.layer(SetRequestIdLayer::x_request_id(MakeRequestUuid))
|
||||
.layer(
|
||||
TraceLayer::new_for_http()
|
||||
.make_span_with(DefaultMakeSpan::new().include_headers(false))
|
||||
.on_response(DefaultOnResponse::new().latency_unit(tower_http::LatencyUnit::Millis)),
|
||||
)
|
||||
.layer(PropagateRequestIdLayer::x_request_id())
|
||||
.layer(TimeoutLayer::new(Duration::from_secs(30)))
|
||||
.layer(CompressionLayer::new())
|
||||
.layer(CorsLayer::permissive()), // tighten in production
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
Order matters: outermost layer wraps the request first. Trace before timeout so timeouts get logged. Compression after trace so trace sees the original body size.
|
||||
|
||||
## Main + graceful shutdown
|
||||
|
||||
```rust
|
||||
// src/main.rs
|
||||
use my_app::{config::Settings, routes, state::AppState};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
init_tracing();
|
||||
|
||||
let config = Settings::load()?;
|
||||
let state = AppState::new(config.clone()).await?;
|
||||
let app = routes::router(state);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind(&config.bind).await?;
|
||||
tracing::info!(addr = %config.bind, "listening");
|
||||
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(shutdown_signal())
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn init_tracing() {
|
||||
use tracing_subscriber::{fmt, EnvFilter};
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("info,sqlx=warn,hyper=warn,tower_http=info"));
|
||||
fmt().with_env_filter(filter).with_target(false).json().init();
|
||||
}
|
||||
|
||||
async fn shutdown_signal() {
|
||||
let ctrl_c = async { tokio::signal::ctrl_c().await.expect("ctrl_c handler"); };
|
||||
#[cfg(unix)]
|
||||
let terminate = async {
|
||||
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
|
||||
.expect("signal handler").recv().await;
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let terminate = std::future::pending::<()>();
|
||||
|
||||
tokio::select! { _ = ctrl_c => {}, _ = terminate => {} }
|
||||
tracing::info!("shutting down");
|
||||
}
|
||||
```
|
||||
|
||||
## Middleware: bearer auth example
|
||||
|
||||
```rust
|
||||
// src/middleware/auth.rs
|
||||
use axum::{
|
||||
extract::{Request, State},
|
||||
http::header::AUTHORIZATION,
|
||||
middleware::Next,
|
||||
response::Response,
|
||||
};
|
||||
use crate::{error::AppError, state::AppState};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AuthUser { pub id: uuid::Uuid }
|
||||
|
||||
pub async fn require_auth(
|
||||
State(state): State<AppState>,
|
||||
mut request: Request,
|
||||
next: Next,
|
||||
) -> Result<Response, AppError> {
|
||||
let token = request.headers()
|
||||
.get(AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|s| s.strip_prefix("Bearer "))
|
||||
.ok_or(AppError::Unauthorized)?;
|
||||
|
||||
let claims = verify_jwt(token, &state.config.jwt_secret)?;
|
||||
request.extensions_mut().insert(AuthUser { id: claims.sub });
|
||||
Ok(next.run(request).await)
|
||||
}
|
||||
```
|
||||
|
||||
Apply with `.route_layer(middleware::from_fn_with_state(state.clone(), require_auth))` on the subroutes that need it.
|
||||
|
||||
## Testing handlers
|
||||
|
||||
```rust
|
||||
// tests/users.rs
|
||||
#[tokio::test]
|
||||
async fn creates_user() {
|
||||
let pool = test_db().await; // helper that spins up a transactional DB
|
||||
let state = AppState::new_test(pool).await.unwrap();
|
||||
let app = my_app::routes::router(state);
|
||||
|
||||
let request = axum::http::Request::builder()
|
||||
.uri("/api/v1/users")
|
||||
.method("POST")
|
||||
.header("content-type", "application/json")
|
||||
.body(axum::body::Body::from(
|
||||
serde_json::to_vec(&serde_json::json!({"email": "a@b.com", "name": "A"})).unwrap()
|
||||
)).unwrap();
|
||||
|
||||
let response = tower::ServiceExt::oneshot(app, request).await.unwrap();
|
||||
assert_eq!(response.status(), 201);
|
||||
let bytes = axum::body::to_bytes(response.into_body(), 1 << 20).await.unwrap();
|
||||
let user: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
|
||||
assert_eq!(user["email"], "a@b.com");
|
||||
}
|
||||
```
|
||||
|
||||
`tower::ServiceExt::oneshot` calls the router directly without binding a socket. Tests run in parallel without port collisions.
|
||||
|
||||
## Config
|
||||
|
||||
```rust
|
||||
// src/config.rs
|
||||
use secrecy::{Secret, ExposeSecret};
|
||||
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
pub struct Settings {
|
||||
pub bind: String,
|
||||
pub db: Database,
|
||||
pub jwt_secret: Secret<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
pub struct Database {
|
||||
pub url: Secret<String>,
|
||||
pub max_connections: u32,
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
pub fn load() -> anyhow::Result<Self> {
|
||||
let cfg = config::Config::builder()
|
||||
.add_source(config::File::with_name("config/default").required(false))
|
||||
.add_source(config::File::with_name(&format!(
|
||||
"config/{}", std::env::var("APP_ENV").unwrap_or_else(|_| "dev".into())
|
||||
)).required(false))
|
||||
.add_source(config::Environment::with_prefix("APP").separator("__"))
|
||||
.build()?;
|
||||
Ok(cfg.try_deserialize()?)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`Secret<T>` from the `secrecy` crate hides the value in `Debug`/`Display` to prevent accidental log leakage. Access via `.expose_secret()` only where needed.
|
||||
|
||||
## OpenAPI (optional)
|
||||
|
||||
Add `utoipa` derive macros on your DTOs and handlers, mount Swagger UI at `/swagger-ui`:
|
||||
|
||||
```rust
|
||||
use utoipa::OpenApi;
|
||||
use utoipa_axum::router::OpenApiRouter;
|
||||
use utoipa_swagger_ui::SwaggerUi;
|
||||
|
||||
#[derive(OpenApi)]
|
||||
#[openapi(
|
||||
paths(routes::users::create_user, routes::users::get_user),
|
||||
components(schemas(routes::users::User, routes::users::CreateUser))
|
||||
)]
|
||||
struct ApiDoc;
|
||||
|
||||
let (router, api) = OpenApiRouter::with_openapi(ApiDoc::openapi())
|
||||
.routes(utoipa_axum::routes!(routes::users::create_user, routes::users::get_user))
|
||||
.split_for_parts();
|
||||
|
||||
let app = router.merge(SwaggerUi::new("/swagger-ui").url("/api-docs/openapi.json", api));
|
||||
```
|
||||
|
||||
## Production checklist
|
||||
|
||||
- Bind to `0.0.0.0` in containers, `127.0.0.1` for local-only services.
|
||||
- Set `RUST_LOG=info,sqlx=warn` (or use `EnvFilter` defaults as shown).
|
||||
- Send logs to stdout in JSON. Ingest via Vector / Fluent Bit / Loki.
|
||||
- Run migrations on startup (`sqlx::migrate!` block). Fail fast on schema mismatch.
|
||||
- Health endpoint **must hit the DB** (so load balancers know if the pool is dead).
|
||||
- Add `tower::limit::RateLimitLayer` or token-bucket middleware for public endpoints.
|
||||
- Set `tower_http::limit::RequestBodyLimitLayer` to bound request size.
|
||||
- Compress with brotli + gzip via `CompressionLayer`.
|
||||
- Tighten CORS, do not ship `CorsLayer::permissive()` to production.
|
||||
- Strip sensitive headers from traces via `SetSensitiveHeadersLayer`.
|
||||
- Set up SIGTERM-driven `with_graceful_shutdown` so deploys roll without dropping requests.
|
||||
- Containerize with `cargo chef` for incremental Docker builds.
|
||||
|
||||
## Common mistakes
|
||||
|
||||
1. **Forgetting `error_for_status()?` on outbound `reqwest`** — 4xx silently succeeds.
|
||||
2. **Returning `Result<T, sqlx::Error>` from handlers** — leak DB details to clients. Always go through `AppError`.
|
||||
3. **`Json<T>` extractor before validation** — invalid JSON returns axum's default 422 with no body shape. Wrap in a `ValidatedJson<T>` extractor that runs `validator` and returns `AppError`.
|
||||
4. **Holding DB connections across `.await` on slow external calls** — exhausts the pool. Acquire late, release early.
|
||||
5. **Skipping `tracing::instrument`** on handlers — losing per-request span correlation.
|
||||
6. **No `RequestBodyLimitLayer`** — DoS surface. Default axum has no limit.
|
||||
@@ -0,0 +1,317 @@
|
||||
# Cargo Strict Configuration
|
||||
|
||||
The exact knobs every new Rust project gets. Drop these in unmodified.
|
||||
|
||||
## `rust-toolchain.toml`
|
||||
|
||||
```toml
|
||||
[toolchain]
|
||||
channel = "stable"
|
||||
components = ["rustfmt", "clippy", "rust-src"]
|
||||
profile = "default"
|
||||
```
|
||||
|
||||
Pin nightly separately when miri runs:
|
||||
|
||||
```bash
|
||||
rustup install nightly
|
||||
rustup component add miri rust-src --toolchain nightly
|
||||
```
|
||||
|
||||
## `Cargo.toml` — `[lints]` section
|
||||
|
||||
```toml
|
||||
[lints.rust]
|
||||
unsafe_op_in_unsafe_fn = "deny"
|
||||
missing_docs = "warn"
|
||||
missing_debug_implementations = "warn"
|
||||
unreachable_pub = "warn"
|
||||
unused_must_use = "deny"
|
||||
elided_lifetimes_in_paths = "warn"
|
||||
non_ascii_idents = "deny"
|
||||
trivial_numeric_casts = "warn"
|
||||
unused_lifetimes = "warn"
|
||||
single_use_lifetimes = "warn"
|
||||
|
||||
[lints.clippy]
|
||||
# Groups
|
||||
all = { level = "deny", priority = -1 }
|
||||
pedantic = { level = "warn", priority = -1 }
|
||||
nursery = { level = "warn", priority = -1 }
|
||||
cargo = { level = "warn", priority = -1 }
|
||||
|
||||
# Hard denies - turn warnings into errors for sharp tools
|
||||
undocumented_unsafe_blocks = "deny"
|
||||
multiple_unsafe_ops_per_block = "deny"
|
||||
unwrap_used = "deny"
|
||||
expect_used = "deny"
|
||||
panic = "deny"
|
||||
todo = "deny"
|
||||
unimplemented = "deny"
|
||||
unreachable = "deny"
|
||||
indexing_slicing = "deny"
|
||||
mem_forget = "deny"
|
||||
arithmetic_side_effects = "warn"
|
||||
cast_possible_truncation = "warn"
|
||||
cast_possible_wrap = "warn"
|
||||
cast_precision_loss = "warn"
|
||||
cast_sign_loss = "warn"
|
||||
as_underscore = "deny"
|
||||
as_ptr_cast_mut = "deny"
|
||||
ptr_as_ptr = "warn"
|
||||
borrow_as_ptr = "warn"
|
||||
fn_to_numeric_cast_any = "deny"
|
||||
clone_on_ref_ptr = "warn"
|
||||
mutex_atomic = "warn"
|
||||
rc_buffer = "warn"
|
||||
rc_mutex = "warn"
|
||||
exit = "warn"
|
||||
allow_attributes_without_reason = "warn"
|
||||
dbg_macro = "warn"
|
||||
print_stderr = "warn"
|
||||
print_stdout = "warn"
|
||||
use_debug = "warn"
|
||||
|
||||
# Stylistic relaxations (project-wide opinions only)
|
||||
module_name_repetitions = "allow"
|
||||
must_use_candidate = "allow"
|
||||
missing_errors_doc = "allow" # we use anyhow::Result with .context() everywhere; doc rule is noisy
|
||||
|
||||
# Restriction lints - opt-in soundness rails
|
||||
unreachable = "deny"
|
||||
mod_module_files = "warn" # prefer foo.rs over foo/mod.rs
|
||||
empty_drop = "warn"
|
||||
empty_structs_with_brackets = "warn"
|
||||
empty_enum = "warn"
|
||||
exhaustive_enums = "warn" # public enums should consider #[non_exhaustive]
|
||||
exhaustive_structs = "warn"
|
||||
```
|
||||
|
||||
The `priority = -1` trick: group-level levels are weak; specific lints below them win. This lets us deny `unwrap_used` while still allowing `pedantic` group warnings instead of denies.
|
||||
|
||||
## `Cargo.toml` — release profile
|
||||
|
||||
```toml
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
lto = "fat"
|
||||
codegen-units = 1
|
||||
strip = "symbols"
|
||||
panic = "abort" # smaller, faster - if you need unwinding (FFI catch), set "unwind"
|
||||
debug = "line-tables-only"
|
||||
|
||||
[profile.dev]
|
||||
opt-level = 0
|
||||
debug = true
|
||||
incremental = true
|
||||
codegen-units = 256
|
||||
split-debuginfo = "unpacked"
|
||||
|
||||
# A profile for miri - opt-level 1 keeps simulation bearable while still
|
||||
# exercising real codegen patterns. miri ignores most profile keys but reads
|
||||
# overflow-checks.
|
||||
[profile.miri]
|
||||
inherits = "test"
|
||||
opt-level = 1
|
||||
overflow-checks = true
|
||||
```
|
||||
|
||||
## `Cargo.toml` — workspace level
|
||||
|
||||
```toml
|
||||
[workspace]
|
||||
resolver = "3"
|
||||
|
||||
[workspace.package]
|
||||
edition = "2024"
|
||||
rust-version = "1.83" # bump only when a needed feature lands
|
||||
license = "Apache-2.0 OR MIT"
|
||||
|
||||
[workspace.lints]
|
||||
# Then in each member crate:
|
||||
# [lints]
|
||||
# workspace = true
|
||||
```
|
||||
|
||||
## `rustfmt.toml`
|
||||
|
||||
```toml
|
||||
edition = "2024"
|
||||
max_width = 100
|
||||
imports_granularity = "Module"
|
||||
group_imports = "StdExternalCrate"
|
||||
reorder_imports = true
|
||||
reorder_modules = true
|
||||
newline_style = "Unix"
|
||||
use_field_init_shorthand = true
|
||||
use_try_shorthand = true
|
||||
unstable_features = false
|
||||
```
|
||||
|
||||
Most options come from stable rustfmt. `imports_granularity` and `group_imports` are nightly-only but ignored cleanly on stable; CI runs `cargo +nightly fmt --check` for the import grouping.
|
||||
|
||||
## `clippy.toml`
|
||||
|
||||
```toml
|
||||
# Reduce cognitive load thresholds.
|
||||
cognitive-complexity-threshold = 25
|
||||
type-complexity-threshold = 250
|
||||
too-many-arguments-threshold = 6
|
||||
too-many-lines-threshold = 100
|
||||
|
||||
# msrv - keeps clippy from suggesting features past our MSRV
|
||||
msrv = "1.83"
|
||||
|
||||
# Avoid `panic` lint complaining about derived Debug impls calling unreachable_unchecked etc.
|
||||
allow-unwrap-in-tests = true
|
||||
allow-expect-in-tests = true
|
||||
allow-panic-in-tests = true
|
||||
allow-dbg-in-tests = true
|
||||
allow-print-in-tests = true
|
||||
|
||||
# Force named arguments above N params
|
||||
single-char-binding-names-threshold = 4
|
||||
```
|
||||
|
||||
## `deny.toml` (cargo-deny)
|
||||
|
||||
```toml
|
||||
[advisories]
|
||||
db-path = "~/.cargo/advisory-db"
|
||||
db-urls = ["https://github.com/rustsec/advisory-db"]
|
||||
yanked = "deny"
|
||||
ignore = []
|
||||
|
||||
[licenses]
|
||||
allow = [
|
||||
"MIT",
|
||||
"Apache-2.0",
|
||||
"Apache-2.0 WITH LLVM-exception",
|
||||
"BSD-2-Clause",
|
||||
"BSD-3-Clause",
|
||||
"ISC",
|
||||
"Unicode-DFS-2016",
|
||||
"Unicode-3.0",
|
||||
"Zlib",
|
||||
"MPL-2.0",
|
||||
"CC0-1.0",
|
||||
]
|
||||
confidence-threshold = 0.93
|
||||
exceptions = []
|
||||
|
||||
[bans]
|
||||
multiple-versions = "warn"
|
||||
wildcards = "deny"
|
||||
highlight = "all"
|
||||
deny = [
|
||||
# Pin out unmaintained alternatives
|
||||
{ name = "async-std", reason = "use tokio" },
|
||||
{ name = "actix-web", reason = "use axum" },
|
||||
{ name = "chrono", reason = "use jiff" },
|
||||
]
|
||||
|
||||
[sources]
|
||||
unknown-registry = "deny"
|
||||
unknown-git = "warn"
|
||||
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
|
||||
```
|
||||
|
||||
## CI Workflow (`.github/workflows/ci.yml`)
|
||||
|
||||
```yaml
|
||||
name: ci
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
fmt:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
with:
|
||||
components: rustfmt
|
||||
- run: cargo +nightly fmt --all -- --check
|
||||
|
||||
clippy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: clippy
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: cargo clippy --all-targets --all-features --workspace -- -D warnings
|
||||
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- run: cargo nextest run --all-targets --all-features --workspace
|
||||
|
||||
miri:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
with:
|
||||
components: miri, rust-src
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- env:
|
||||
MIRIFLAGS: "-Zmiri-strict-provenance -Zmiri-symbolic-alignment-check"
|
||||
run: cargo +nightly miri nextest run --all-features --workspace
|
||||
|
||||
machete:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: bnjbvr/cargo-machete@main
|
||||
|
||||
deny:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: EmbarkStudios/cargo-deny-action@v2
|
||||
with:
|
||||
command: check all
|
||||
|
||||
audit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: rustsec/audit-check@v1
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
```
|
||||
|
||||
## Project bootstrap
|
||||
|
||||
```bash
|
||||
cargo new --bin my-app --edition 2024
|
||||
cd my-app
|
||||
cargo install cargo-nextest cargo-machete cargo-deny cargo-edit cargo-watch
|
||||
rustup install nightly
|
||||
rustup component add miri --toolchain nightly
|
||||
# drop the configs above
|
||||
git add . && git commit -m "chore: bootstrap strict toolchain"
|
||||
```
|
||||
|
||||
After every change:
|
||||
|
||||
```bash
|
||||
cargo fmt --all -- --check && \
|
||||
cargo clippy --all-targets --all-features -- -D warnings && \
|
||||
cargo nextest run && \
|
||||
cargo +nightly miri nextest run # only if unsafe is involved
|
||||
```
|
||||
@@ -0,0 +1,409 @@
|
||||
# CLI Stack — clap + color-eyre + tracing + indicatif + dialoguer
|
||||
|
||||
The default for any new CLI tool. Strict typing on arguments, beautiful errors, progress feedback, interactive prompts when needed.
|
||||
|
||||
## Cargo.toml
|
||||
|
||||
```toml
|
||||
[package]
|
||||
name = "mytool"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
clap = { version = "4", features = ["derive", "env", "wrap_help", "color", "unicode"] }
|
||||
clap_complete = "4"
|
||||
color-eyre = "0.6"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
|
||||
anyhow = "1"
|
||||
indicatif = { version = "0.17", features = ["tokio"] }
|
||||
dialoguer = { version = "0.11", features = ["fuzzy-select"] }
|
||||
console = "0.15"
|
||||
tokio = { version = "1", features = ["macros", "rt-multi-thread", "fs", "process", "signal"] }
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
lto = "fat"
|
||||
codegen-units = 1
|
||||
strip = "symbols"
|
||||
panic = "abort"
|
||||
```
|
||||
|
||||
## Command structure
|
||||
|
||||
```rust
|
||||
// src/cli.rs
|
||||
use clap::{Parser, Subcommand, ValueEnum};
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(
|
||||
name = "mytool",
|
||||
author,
|
||||
version,
|
||||
about = "A short description",
|
||||
long_about = "A longer description that appears in --help",
|
||||
arg_required_else_help = true,
|
||||
)]
|
||||
pub struct Cli {
|
||||
/// Configuration file path
|
||||
#[arg(short, long, env = "MYTOOL_CONFIG", default_value = "config.toml", global = true)]
|
||||
pub config: PathBuf,
|
||||
|
||||
/// Increase verbosity (-v info, -vv debug, -vvv trace)
|
||||
#[arg(short, long, action = clap::ArgAction::Count, global = true)]
|
||||
pub verbose: u8,
|
||||
|
||||
/// Suppress all non-error output
|
||||
#[arg(short, long, global = true, conflicts_with = "verbose")]
|
||||
pub quiet: bool,
|
||||
|
||||
/// Force colored output even when stdout is not a terminal
|
||||
#[arg(long, global = true, value_enum, default_value_t = ColorChoice::Auto)]
|
||||
pub color: ColorChoice,
|
||||
|
||||
/// Output format
|
||||
#[arg(short, long, global = true, value_enum, default_value_t = OutputFormat::Pretty)]
|
||||
pub format: OutputFormat,
|
||||
|
||||
#[command(subcommand)]
|
||||
pub command: Command,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, ValueEnum)]
|
||||
pub enum ColorChoice { Auto, Always, Never }
|
||||
|
||||
#[derive(Debug, Clone, ValueEnum)]
|
||||
pub enum OutputFormat { Pretty, Json, Plain }
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
pub enum Command {
|
||||
/// Build the thing
|
||||
Build(BuildArgs),
|
||||
/// Watch and rebuild
|
||||
Watch(WatchArgs),
|
||||
/// Generate shell completions
|
||||
Completions { #[arg(value_enum)] shell: clap_complete::Shell },
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
pub struct BuildArgs {
|
||||
/// Target directory
|
||||
#[arg(short, long, default_value = "target")]
|
||||
pub target: PathBuf,
|
||||
/// Build mode
|
||||
#[arg(short, long, value_enum, default_value_t = Mode::Release)]
|
||||
pub mode: Mode,
|
||||
/// Specific files to build (default: all)
|
||||
pub files: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
pub struct WatchArgs {
|
||||
/// Glob pattern to watch
|
||||
#[arg(short, long, default_value = "**/*.rs")]
|
||||
pub pattern: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, ValueEnum)]
|
||||
pub enum Mode { Debug, Release }
|
||||
```
|
||||
|
||||
Key clap derive patterns:
|
||||
|
||||
- `env = "VAR"` — falls back to env var if flag not given.
|
||||
- `global = true` — flag inherits to subcommands.
|
||||
- `arg_required_else_help = true` — running with no args prints help instead of erroring.
|
||||
- `value_enum` on an enum — case-insensitive parsing + auto-completion.
|
||||
- `action = clap::ArgAction::Count` — `-v` is 1, `-vv` is 2, etc.
|
||||
- `conflicts_with` — incompatible flags.
|
||||
|
||||
## Main + tracing init
|
||||
|
||||
```rust
|
||||
// src/main.rs
|
||||
use clap::Parser;
|
||||
use mytool::cli::{Cli, Command, ColorChoice};
|
||||
use tracing::Level;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
let cli = Cli::parse();
|
||||
init_tracing(&cli);
|
||||
|
||||
if matches!(cli.color, ColorChoice::Always) {
|
||||
console::set_colors_enabled(true);
|
||||
} else if matches!(cli.color, ColorChoice::Never) {
|
||||
console::set_colors_enabled(false);
|
||||
}
|
||||
|
||||
match cli.command {
|
||||
Command::Build(args) => mytool::commands::build::run(&cli, args),
|
||||
Command::Watch(args) => mytool::commands::watch::run(&cli, args),
|
||||
Command::Completions { shell } => {
|
||||
let mut cmd = <Cli as clap::CommandFactory>::command();
|
||||
clap_complete::generate(shell, &mut cmd, "mytool", &mut std::io::stdout());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn init_tracing(cli: &Cli) {
|
||||
let level = if cli.quiet {
|
||||
Level::ERROR
|
||||
} else {
|
||||
match cli.verbose {
|
||||
0 => Level::WARN,
|
||||
1 => Level::INFO,
|
||||
2 => Level::DEBUG,
|
||||
_ => Level::TRACE,
|
||||
}
|
||||
};
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new(format!("mytool={level}")));
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(filter)
|
||||
.with_target(false)
|
||||
.without_time()
|
||||
.compact()
|
||||
.with_writer(std::io::stderr)
|
||||
.init();
|
||||
}
|
||||
```
|
||||
|
||||
Tracing on a CLI:
|
||||
- **Write to stderr.** stdout is for the tool's actual output (which the user might pipe). Logs and progress bars go to stderr.
|
||||
- **Verbosity from `-v`, not from `RUST_LOG`.** Users expect `-v` on a CLI; `RUST_LOG` is a developer escape hatch (kept, but secondary).
|
||||
|
||||
## Progress bars — `indicatif`
|
||||
|
||||
```rust
|
||||
use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
|
||||
use std::time::Duration;
|
||||
|
||||
let mp = MultiProgress::new();
|
||||
let pb = mp.add(ProgressBar::new(files.len() as u64));
|
||||
pb.set_style(
|
||||
ProgressStyle::with_template(
|
||||
"{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len} ({eta}) {msg}"
|
||||
)?
|
||||
.progress_chars("=>-")
|
||||
);
|
||||
|
||||
for file in files {
|
||||
pb.set_message(file.display().to_string());
|
||||
process(&file)?;
|
||||
pb.inc(1);
|
||||
}
|
||||
pb.finish_with_message("done");
|
||||
```
|
||||
|
||||
For unbounded operations:
|
||||
|
||||
```rust
|
||||
let spinner = ProgressBar::new_spinner();
|
||||
spinner.enable_steady_tick(Duration::from_millis(80));
|
||||
spinner.set_message("connecting…");
|
||||
let result = connect().await?;
|
||||
spinner.finish_and_clear();
|
||||
```
|
||||
|
||||
With multiple parallel tasks:
|
||||
|
||||
```rust
|
||||
let mp = MultiProgress::new();
|
||||
let bars: Vec<_> = (0..workers).map(|i| {
|
||||
let pb = mp.add(ProgressBar::new(unit));
|
||||
pb.set_style(ProgressStyle::with_template("worker {prefix}: {pos}/{len}")?);
|
||||
pb.set_prefix(i.to_string());
|
||||
pb
|
||||
}).collect();
|
||||
```
|
||||
|
||||
`MultiProgress` keeps bars stacked and redraws cleanly even with concurrent updates from multiple tasks.
|
||||
|
||||
When stdout is not a terminal, indicatif silently disables animation. Force on/off with `pb.set_draw_target(ProgressDrawTarget::stdout())` / `hidden()`.
|
||||
|
||||
## Interactive prompts — `dialoguer`
|
||||
|
||||
```rust
|
||||
use dialoguer::{theme::ColorfulTheme, Confirm, Input, Password, Select, FuzzySelect, MultiSelect};
|
||||
|
||||
let name: String = Input::with_theme(&ColorfulTheme::default())
|
||||
.with_prompt("Project name")
|
||||
.validate_with(|input: &String| -> Result<(), &str> {
|
||||
if input.chars().all(|c| c.is_alphanumeric() || c == '-' || c == '_') {
|
||||
Ok(())
|
||||
} else {
|
||||
Err("alphanumeric, dash, underscore only")
|
||||
}
|
||||
})
|
||||
.interact_text()?;
|
||||
|
||||
let secret = Password::with_theme(&ColorfulTheme::default())
|
||||
.with_prompt("API key")
|
||||
.with_confirmation("Repeat", "passwords don't match")
|
||||
.interact()?;
|
||||
|
||||
let go: bool = Confirm::with_theme(&ColorfulTheme::default())
|
||||
.with_prompt(format!("Delete {}? This cannot be undone.", path.display()))
|
||||
.default(false)
|
||||
.interact()?;
|
||||
if !go { return Ok(()); }
|
||||
|
||||
let items = ["yes", "no", "maybe"];
|
||||
let idx = Select::with_theme(&ColorfulTheme::default())
|
||||
.with_prompt("Pick one")
|
||||
.items(&items)
|
||||
.default(0)
|
||||
.interact()?;
|
||||
|
||||
let picks = MultiSelect::with_theme(&ColorfulTheme::default())
|
||||
.with_prompt("Toggle features")
|
||||
.items(&["alpha", "beta", "gamma"])
|
||||
.defaults(&[true, false, false])
|
||||
.interact()?;
|
||||
```
|
||||
|
||||
Detect non-TTY before prompting:
|
||||
|
||||
```rust
|
||||
if !console::user_attended() {
|
||||
return Err(anyhow::anyhow!("input required but stdin is not a terminal"));
|
||||
}
|
||||
```
|
||||
|
||||
For automated tests, expose a `--non-interactive` flag and gate all prompts behind it.
|
||||
|
||||
## Structured output
|
||||
|
||||
```rust
|
||||
match cli.format {
|
||||
OutputFormat::Json => {
|
||||
serde_json::to_writer(std::io::stdout().lock(), &result)?;
|
||||
println!();
|
||||
}
|
||||
OutputFormat::Plain => {
|
||||
for row in &result.rows {
|
||||
println!("{}\t{}\t{}", row.a, row.b, row.c);
|
||||
}
|
||||
}
|
||||
OutputFormat::Pretty => {
|
||||
use console::{style, Term};
|
||||
let term = Term::stdout();
|
||||
for row in &result.rows {
|
||||
term.write_line(&format!(
|
||||
"{} {} {}",
|
||||
style(&row.a).green(),
|
||||
style(&row.b).yellow(),
|
||||
style(&row.c).dim(),
|
||||
))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Always offer `--format json` for piping into `jq`, scripts, and other tools.
|
||||
|
||||
## Shell completions
|
||||
|
||||
Already shown in the `Completions` subcommand above. Distribute completions by adding to the install script:
|
||||
|
||||
```bash
|
||||
mytool completions bash > /etc/bash_completion.d/mytool
|
||||
mytool completions fish > ~/.config/fish/completions/mytool.fish
|
||||
mytool completions zsh > "${fpath[1]}/_mytool"
|
||||
```
|
||||
|
||||
## Signal handling
|
||||
|
||||
```rust
|
||||
// In an async CLI command
|
||||
use tokio::signal::ctrl_c;
|
||||
|
||||
tokio::select! {
|
||||
_ = ctrl_c() => {
|
||||
tracing::warn!("interrupted, cleaning up");
|
||||
cleanup().await?;
|
||||
std::process::exit(130); // standard exit code for SIGINT
|
||||
}
|
||||
result = long_running_task() => {
|
||||
result
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
For sync CLIs, install a one-shot handler with `ctrlc` crate:
|
||||
|
||||
```rust
|
||||
let interrupted = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let i = interrupted.clone();
|
||||
ctrlc::set_handler(move || i.store(true, std::sync::atomic::Ordering::SeqCst))?;
|
||||
|
||||
while !interrupted.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
do_step()?;
|
||||
}
|
||||
```
|
||||
|
||||
## Error reporting with color-eyre
|
||||
|
||||
```rust
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::config::HookBuilder::default()
|
||||
.display_env_section(false) // hide SPANTRACE/BACKTRACE env hints by default
|
||||
.display_location_section(false) // hide file:line section
|
||||
.panic_section("If this is a bug, please report at https://github.com/me/mytool/issues")
|
||||
.install()?;
|
||||
real_main()
|
||||
}
|
||||
```
|
||||
|
||||
Errors with `.wrap_err("...")` from `eyre::WrapErr` (compatible with anyhow's `.context`) show as a numbered chain. `RUST_BACKTRACE=1` shows the full trace; `RUST_SPANTRACE=1` shows tracing spans where the error fired.
|
||||
|
||||
## Distribution
|
||||
|
||||
- Add `cargo dist init` for prebuilt binary release pipeline (cross-platform tarballs + installers).
|
||||
- Publish to Homebrew tap, AUR, scoop, Chocolatey via dist.
|
||||
- Sign Linux binaries with `cosign` if your audience is enterprise.
|
||||
- Build single static binary on Linux with `--target x86_64-unknown-linux-musl` (or `aarch64-unknown-linux-musl`).
|
||||
- For wasm-runnable CLIs (`wasi-cli`), add `--target wasm32-wasip1`.
|
||||
|
||||
## Common mistakes
|
||||
|
||||
1. **Mixing stdout and stderr.** Tool output goes to stdout; logs and progress go to stderr.
|
||||
2. **No `--non-interactive` flag.** Interactive prompts block automation.
|
||||
3. **Printing colored output unconditionally.** Honor `NO_COLOR` env var, detect TTY with `console::user_attended()`.
|
||||
4. **`println!` for errors.** Use `tracing::error!` so logs go to stderr automatically and respect verbosity.
|
||||
5. **`unwrap()` on `Cli::parse()`.** clap returns clean errors with `--help` text; `parse()` exits on its own.
|
||||
6. **Long subcommand handlers in `main.rs`.** Split into `src/commands/<name>.rs` per command.
|
||||
7. **Missing exit code semantics.** Use `std::process::exit(1)` (general error), `2` (usage), `130` (SIGINT) appropriately. Or return `Result` and let main map.
|
||||
|
||||
## Testing CLIs
|
||||
|
||||
```rust
|
||||
// tests/cli.rs
|
||||
use assert_cmd::Command;
|
||||
use predicates::prelude::*;
|
||||
|
||||
#[test]
|
||||
fn shows_help() {
|
||||
Command::cargo_bin("mytool").unwrap()
|
||||
.arg("--help")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("Usage:"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_unknown_subcommand() {
|
||||
Command::cargo_bin("mytool").unwrap()
|
||||
.arg("nope")
|
||||
.assert()
|
||||
.failure()
|
||||
.stderr(predicate::str::contains("unrecognized subcommand"));
|
||||
}
|
||||
```
|
||||
|
||||
`assert_cmd` builds the binary once per test run and gives a fluent assertion API.
|
||||
@@ -0,0 +1,375 @@
|
||||
# Concurrency Primitives
|
||||
|
||||
Locks, atomics, channels, and the loom model checker. The decision tree that keeps the agent out of soundness trouble.
|
||||
|
||||
## The pyramid
|
||||
|
||||
```
|
||||
Highest level tokio::sync::mpsc / broadcast / watch
|
||||
(message passing — default for new code)
|
||||
|
||||
Arc<Mutex<T>> / Arc<RwLock<T>>
|
||||
(shared mutable state — common, easy to get right)
|
||||
|
||||
parking_lot::{Mutex, RwLock, Condvar}
|
||||
(faster sync locks, no poisoning)
|
||||
|
||||
Atomics (AtomicUsize, AtomicBool, AtomicPtr)
|
||||
(single-word lock-free state)
|
||||
|
||||
Lowest level UnsafeCell + unsafe + loom + miri
|
||||
(custom lock-free / wait-free primitives)
|
||||
```
|
||||
|
||||
**Always start at the top.** Drop a level only when you have measured a real bottleneck.
|
||||
|
||||
## Decision tree
|
||||
|
||||
```
|
||||
Need to share state between tasks?
|
||||
├── State is configuration (read-only after start)
|
||||
│ └── Arc<Config> (no lock needed)
|
||||
├── State is a queue of work
|
||||
│ └── tokio::sync::mpsc::channel(cap)
|
||||
├── State is "latest value" published to many readers
|
||||
│ └── tokio::sync::watch::channel(initial)
|
||||
├── State is broadcast (every consumer sees every value)
|
||||
│ └── tokio::sync::broadcast::channel(cap)
|
||||
├── State is request-response within one task tree
|
||||
│ └── tokio::sync::oneshot::channel()
|
||||
├── State is a counter
|
||||
│ └── AtomicU64 (or AtomicUsize)
|
||||
├── State is a flag / set-once
|
||||
│ └── AtomicBool / OnceLock<T> / OnceCell<T>
|
||||
├── State needs mutation across many tasks/threads, cheap critical sections
|
||||
│ ├── async context → tokio::sync::Mutex<T>
|
||||
│ └── sync context (no .await held) → parking_lot::Mutex<T>
|
||||
├── State needs mutation, many readers, few writers
|
||||
│ ├── async context → tokio::sync::RwLock<T>
|
||||
│ └── sync context → parking_lot::RwLock<T>
|
||||
└── State is a custom lock-free primitive (channels, hazard pointers)
|
||||
└── UnsafeCell + atomics + loom-tested + miri-tested + a co-author
|
||||
```
|
||||
|
||||
## Atomics — when and how
|
||||
|
||||
Use atomics for:
|
||||
- Counters incremented from many threads (`AtomicU64`).
|
||||
- Single-shot flags (`AtomicBool`).
|
||||
- Pointer publication (`AtomicPtr<T>`).
|
||||
|
||||
### Memory orderings
|
||||
|
||||
```rust
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let c = AtomicUsize::new(0);
|
||||
|
||||
// Just need a count, no synchronization with other data
|
||||
c.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
// Reading a counter that was incremented from elsewhere
|
||||
let n = c.load(Ordering::Relaxed);
|
||||
```
|
||||
|
||||
| Ordering | When |
|
||||
|---|---|
|
||||
| `Relaxed` | Standalone counters, no other memory needs to be synchronized. |
|
||||
| `Acquire` (loads) / `Release` (stores) | Publish/consume pattern: you write some data then release a flag, readers acquire the flag then read the data. |
|
||||
| `AcqRel` | RMW that both reads-and-publishes (e.g., `fetch_add` on a sequence number). |
|
||||
| `SeqCst` | Total ordering across all `SeqCst` ops. Strongest, slowest. Use when in doubt and switch to a weaker ordering after testing under loom. |
|
||||
|
||||
**Default to `SeqCst` if unsure.** Performance difference is usually negligible. Going weaker requires loom.
|
||||
|
||||
### Publish-then-load pattern
|
||||
|
||||
```rust
|
||||
static READY: AtomicBool = AtomicBool::new(false);
|
||||
static mut DATA: Option<Config> = None;
|
||||
|
||||
// Producer thread:
|
||||
unsafe { DATA = Some(load_config()); }
|
||||
READY.store(true, Ordering::Release);
|
||||
|
||||
// Consumer thread:
|
||||
if READY.load(Ordering::Acquire) {
|
||||
// SAFETY: producer's Release pairs with our Acquire; if we see READY=true,
|
||||
// we are guaranteed to also see the DATA write that happened-before it.
|
||||
let cfg = unsafe { DATA.as_ref().unwrap() };
|
||||
}
|
||||
```
|
||||
|
||||
This is the canonical Release/Acquire pattern. **Use `OnceLock<Config>` instead** in new code — it encapsulates exactly this with safe API.
|
||||
|
||||
## Std vs parking_lot vs tokio for locks
|
||||
|
||||
| | std::sync::Mutex | parking_lot::Mutex | tokio::sync::Mutex |
|
||||
|---|---|---|---|
|
||||
| Speed | Slowest (OS futex direct) | Fastest (smarter parking) | Slow (await-aware) |
|
||||
| Poisoning | Yes (`PoisonError`) | No | No |
|
||||
| Hold across `.await` | Dangerous (deadlock under current-thread runtime) | Dangerous | Safe |
|
||||
| Drop guard releases | Yes | Yes | Yes |
|
||||
| RAII | Yes (`MutexGuard`) | Yes | Yes |
|
||||
| Const constructor | Yes (since 1.63) | Yes | No |
|
||||
| Async | No | No | Yes |
|
||||
|
||||
**Rule of thumb:**
|
||||
|
||||
- Hot, short critical section, no await inside → `parking_lot::Mutex`.
|
||||
- Shared state held across `.await` → `tokio::sync::Mutex`.
|
||||
- Static init / app config → `OnceLock` or `LazyLock`.
|
||||
- Avoid `std::sync::Mutex` for new code; the poisoning behavior is more annoying than useful and `parking_lot` is strictly faster.
|
||||
|
||||
### Common deadlock — async + sync mutex
|
||||
|
||||
```rust
|
||||
let m = std::sync::Mutex::new(0u64);
|
||||
let guard = m.lock().unwrap();
|
||||
something_async().await; // ❌ guard is held across await
|
||||
*guard += 1;
|
||||
```
|
||||
|
||||
Under `current_thread` runtime this deadlocks (the future suspends while holding the lock; another future on the same thread tries to acquire, blocks the executor). Under `multi_thread` it works but serializes the system.
|
||||
|
||||
Fix:
|
||||
|
||||
```rust
|
||||
{
|
||||
let mut guard = m.lock().unwrap();
|
||||
*guard += 1;
|
||||
} // guard released
|
||||
something_async().await;
|
||||
```
|
||||
|
||||
Or switch to `tokio::sync::Mutex` whose guard is `Send` across awaits.
|
||||
|
||||
## Channels
|
||||
|
||||
### Mpsc — the workhorse
|
||||
|
||||
```rust
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel::<Job>(256);
|
||||
|
||||
tokio::spawn(async move {
|
||||
while let Some(job) = rx.recv().await {
|
||||
process(job).await;
|
||||
}
|
||||
});
|
||||
|
||||
tx.send(Job::new()).await?; // backpressure: awaits if full
|
||||
```
|
||||
|
||||
Capacity is the backpressure budget. **Never `unbounded_channel()`** unless you have a hard upper bound elsewhere; otherwise it is a slow-leak memory bomb.
|
||||
|
||||
### Watch — latest-value pubsub
|
||||
|
||||
```rust
|
||||
let (tx, mut rx) = tokio::sync::watch::channel(Config::default());
|
||||
|
||||
// Producer:
|
||||
tx.send(new_config)?;
|
||||
|
||||
// Consumer:
|
||||
loop {
|
||||
rx.changed().await?;
|
||||
let cfg = rx.borrow();
|
||||
apply(&cfg);
|
||||
}
|
||||
```
|
||||
|
||||
Receivers see only the latest value (older updates are dropped). Perfect for config reload, leadership changes, "current time" propagation.
|
||||
|
||||
### Broadcast — fanout queue
|
||||
|
||||
```rust
|
||||
let (tx, _) = tokio::sync::broadcast::channel::<Event>(1024);
|
||||
let mut rx1 = tx.subscribe();
|
||||
let mut rx2 = tx.subscribe();
|
||||
|
||||
while let Ok(event) = rx1.recv().await {
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
Each subscriber has its own buffer. If a subscriber falls behind by more than the buffer size, it gets `RecvError::Lagged(n)` and skips messages. Decide explicitly: log + continue, or drop the subscriber and reconnect.
|
||||
|
||||
### Oneshot — single value
|
||||
|
||||
```rust
|
||||
let (tx, rx) = tokio::sync::oneshot::channel::<Response>();
|
||||
worker.send(Request { reply: tx }).await?;
|
||||
let response = rx.await?;
|
||||
```
|
||||
|
||||
The standard request/response pattern over an actor.
|
||||
|
||||
## Semaphores
|
||||
|
||||
Bound concurrent operations:
|
||||
|
||||
```rust
|
||||
let sem = Arc::new(tokio::sync::Semaphore::new(10));
|
||||
|
||||
for task in tasks {
|
||||
let permit = sem.clone().acquire_owned().await?;
|
||||
tokio::spawn(async move {
|
||||
let _hold = permit; // released when task exits
|
||||
process(task).await
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
Use cases:
|
||||
- "Max 10 outbound HTTP requests in flight."
|
||||
- "Max 3 DB connections doing writes."
|
||||
- "Max N tokio tasks running heavy CPU."
|
||||
|
||||
A semaphore with `permits=1` is a mutex. Use the actual `Mutex` for that — clearer intent.
|
||||
|
||||
## Arc and Rc
|
||||
|
||||
`Arc<T>` for cross-thread shared ownership, `Rc<T>` for single-thread (never spans threads).
|
||||
|
||||
```rust
|
||||
let shared = Arc::new(BigData::new());
|
||||
for _ in 0..workers {
|
||||
let s = shared.clone();
|
||||
tokio::spawn(async move { use_data(&s).await });
|
||||
}
|
||||
```
|
||||
|
||||
`Arc::clone(&s)` is just a reference-count increment; the data is not copied.
|
||||
|
||||
**Do not clone in hot loops** if you can pass a reference. `&Arc<T>` is fine to pass; only call `Arc::clone` when you need to move ownership across a thread/task boundary.
|
||||
|
||||
`Weak<T>` for back-references in graphs / parent pointers to avoid cycles.
|
||||
|
||||
## Once-init primitives
|
||||
|
||||
```rust
|
||||
use std::sync::{OnceLock, LazyLock};
|
||||
|
||||
// Lazy initialization, computed on first read
|
||||
static CONFIG: LazyLock<Config> = LazyLock::new(|| Config::load_from_env().unwrap());
|
||||
|
||||
fn get_config() -> &'static Config {
|
||||
&CONFIG
|
||||
}
|
||||
|
||||
// One-shot publication, set explicitly
|
||||
static DB: OnceLock<sqlx::PgPool> = OnceLock::new();
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let pool = sqlx::PgPool::connect(&env_url()).await.unwrap();
|
||||
DB.set(pool).expect("only set once");
|
||||
// Now everywhere: DB.get().unwrap()
|
||||
}
|
||||
```
|
||||
|
||||
`OnceLock` is `std::sync` and stable. `LazyLock` is in `std::sync` since 1.80. Avoid the older `once_cell` crate for new code.
|
||||
|
||||
## Loom — model-checking lock-free code
|
||||
|
||||
When `unsafe` participates in a concurrent algorithm, miri's single-thread model is insufficient. Loom exhaustively explores thread interleavings.
|
||||
|
||||
`Cargo.toml`:
|
||||
|
||||
```toml
|
||||
[target.'cfg(loom)'.dev-dependencies]
|
||||
loom = "0.7"
|
||||
```
|
||||
|
||||
In code, switch between real and loom primitives:
|
||||
|
||||
```rust
|
||||
#[cfg(loom)]
|
||||
use loom::sync::atomic::{AtomicUsize, Ordering};
|
||||
#[cfg(not(loom))]
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
#[cfg(loom)]
|
||||
use loom::sync::Arc;
|
||||
#[cfg(not(loom))]
|
||||
use std::sync::Arc;
|
||||
```
|
||||
|
||||
Write a test:
|
||||
|
||||
```rust
|
||||
#[cfg(loom)]
|
||||
mod loom_tests {
|
||||
use super::*;
|
||||
use loom::thread;
|
||||
|
||||
#[test]
|
||||
fn concurrent_push_pop_preserves_order() {
|
||||
loom::model(|| {
|
||||
let queue = Arc::new(MyQueue::new());
|
||||
let q1 = queue.clone();
|
||||
let q2 = queue.clone();
|
||||
let h1 = thread::spawn(move || q1.push(1));
|
||||
let h2 = thread::spawn(move || q2.pop());
|
||||
h1.join().unwrap();
|
||||
h2.join().unwrap();
|
||||
// Assert the invariant: queue is in a coherent state.
|
||||
});
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
RUSTFLAGS="--cfg loom" cargo test --release -- --test-threads 1
|
||||
```
|
||||
|
||||
Loom explores every legal scheduling of the threads, including those a real scheduler would rarely produce. If your code has a race, loom will find it deterministically.
|
||||
|
||||
### Loom's limits
|
||||
|
||||
- Slow. Each `loom::model` invocation explores many schedules; keep tests tiny (2-3 threads, a few operations each).
|
||||
- Single-machine only. Doesn't model distributed systems.
|
||||
- Doesn't catch UB inside `unsafe` blocks the way miri does. **Run both: miri for memory safety, loom for thread schedules.**
|
||||
- Doesn't handle `tokio` directly. Loom replaces stdlib's sync primitives; tokio's are independent.
|
||||
|
||||
## Send and Sync — what they mean
|
||||
|
||||
- `T: Send` — `T` can be moved between threads safely.
|
||||
- `T: Sync` — `&T` can be shared between threads safely.
|
||||
|
||||
These are auto-derived for composite types if all components implement them. Manual `unsafe impl Send/Sync` is required only for raw pointer types and FFI handles.
|
||||
|
||||
```rust
|
||||
struct MyHandle { raw: *mut FfiObject }
|
||||
|
||||
// SAFETY: FfiObject's documented contract states that move-between-threads
|
||||
// is safe as long as concurrent use is externally synchronized. We do not
|
||||
// implement Sync because the FFI object is single-threaded once obtained.
|
||||
unsafe impl Send for MyHandle {}
|
||||
// Do NOT impl Sync — the FFI is not thread-safe.
|
||||
```
|
||||
|
||||
When the compiler complains that "T: Send is not satisfied", the cause is usually a raw pointer, an `Rc` (not `Arc`), or a `RefCell` (use `Mutex`).
|
||||
|
||||
## Common mistakes
|
||||
|
||||
1. **Holding a `std::sync::Mutex` guard across `.await`.** Compiles, deadlocks at runtime under `current_thread`.
|
||||
2. **`Arc::clone` in a tight loop.** Refcount bump is cheap but not free; pass `&Arc<T>` when possible.
|
||||
3. **`Mutex<HashMap<K, V>>` for hot reads.** Switch to `RwLock` or `Arc<dashmap::DashMap>`.
|
||||
4. **Atomic operations with `Ordering::Relaxed` for happens-before publication.** You need `Release`/`Acquire`. Run under loom to be sure.
|
||||
5. **Unbounded channels.** Always set capacity. If you "know it won't backlog", you don't, and it will.
|
||||
6. **Spawning detached tokio tasks for fire-and-forget cleanup.** Use `JoinSet` so panics surface.
|
||||
7. **`std::mem::transmute` to fake `Send`/`Sync`.** Use `unsafe impl` with a SAFETY comment instead. Transmute breaks Stacked Borrows and miri.
|
||||
8. **Locking order inversion across two mutexes.** Always acquire in a globally consistent order. For more than three locks, switch to a single mutex around a struct.
|
||||
|
||||
## When to escape to lock-free
|
||||
|
||||
You should reach for atomics + `UnsafeCell` only when:
|
||||
1. The hot path is **measured** to be bottlenecked on lock contention.
|
||||
2. There is no existing library (crossbeam, atomic-queue, hazardous) that solves your problem.
|
||||
3. You can write loom tests that pass.
|
||||
4. You can write miri tests that pass.
|
||||
5. You have at least one other engineer who can review the algorithm.
|
||||
|
||||
Practically all "I want to write a lock-free queue" projects fail (3) or (4). When in doubt, take the lock and move on.
|
||||
@@ -0,0 +1,439 @@
|
||||
# Library Defaults — Full Decision Tree
|
||||
|
||||
The opinionated, audited-in-prod stack for 2026 Rust. Every entry has a one-line rationale and a canonical code snippet so the agent does not have to relearn each library's idioms.
|
||||
|
||||
## Async runtime — `tokio`
|
||||
|
||||
The default. Use `tokio` for new work. Multi-thread runtime unless you have a measured reason to go single-thread.
|
||||
|
||||
```rust
|
||||
#[tokio::main(flavor = "multi_thread", worker_threads = 8)]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
tracing_subscriber::fmt::init();
|
||||
run().await
|
||||
}
|
||||
```
|
||||
|
||||
Avoid:
|
||||
- `async-std` — unmaintained, last release ages ago. crates.io download counts are misleading because of historical inertia.
|
||||
- `smol` — fine for embedded-ish niches; outside that, the ecosystem is on tokio.
|
||||
- Mixing runtimes in one binary. Pick one and stay.
|
||||
|
||||
## Errors — `anyhow` (apps) + `thiserror` (libs)
|
||||
|
||||
Application boundaries get `anyhow::Error` with `.context("...")` at every layer that adds meaning. Libraries expose `#[derive(thiserror::Error)]` enums with `#[non_exhaustive]`.
|
||||
|
||||
```rust
|
||||
// Application code
|
||||
use anyhow::Context as _;
|
||||
|
||||
pub async fn load_config(path: &Path) -> anyhow::Result<Config> {
|
||||
let text = tokio::fs::read_to_string(path)
|
||||
.await
|
||||
.with_context(|| format!("reading config from {}", path.display()))?;
|
||||
let cfg: Config = toml::from_str(&text)
|
||||
.with_context(|| format!("parsing config at {}", path.display()))?;
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
// Library code
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[non_exhaustive]
|
||||
pub enum ParseError {
|
||||
#[error("expected {expected}, found {found} at position {position}")]
|
||||
Mismatch { expected: &'static str, found: String, position: usize },
|
||||
#[error("unexpected end of input after {context}")]
|
||||
UnexpectedEof { context: &'static str },
|
||||
#[error(transparent)]
|
||||
Io(#[from] std::io::Error),
|
||||
}
|
||||
```
|
||||
|
||||
`#[non_exhaustive]` on enums prevents downstream `match` from breaking when you add variants. `#[error(transparent)]` on a wrapper variant forwards Display + cause to the inner error.
|
||||
|
||||
## CLI — `clap` with derive
|
||||
|
||||
```rust
|
||||
use clap::{Parser, Subcommand};
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(author, version, about, long_about = None)]
|
||||
struct Cli {
|
||||
/// Path to the config file
|
||||
#[arg(short, long, env = "MYAPP_CONFIG", default_value = "config.toml")]
|
||||
config: PathBuf,
|
||||
|
||||
/// Enable verbose output (-v, -vv, -vvv)
|
||||
#[arg(short, long, action = clap::ArgAction::Count)]
|
||||
verbose: u8,
|
||||
|
||||
#[command(subcommand)]
|
||||
command: Command,
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum Command {
|
||||
/// Run the server
|
||||
Serve {
|
||||
#[arg(short, long, default_value_t = 8080)]
|
||||
port: u16,
|
||||
},
|
||||
/// Migrate the database
|
||||
Migrate {
|
||||
#[arg(long)]
|
||||
dry_run: bool,
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
Avoid `structopt` (deprecated, merged into clap), `argh` (less ergonomic), `pico-args` (only when binary size matters more than DX).
|
||||
|
||||
## Logging — `tracing` + `tracing-subscriber`
|
||||
|
||||
Not `log` + `env_logger`. `tracing` supports spans (structured context that follows async tasks) and structured fields - `log` cannot.
|
||||
|
||||
```rust
|
||||
use tracing::{info, instrument, warn, Level};
|
||||
use tracing_subscriber::{fmt, EnvFilter};
|
||||
|
||||
fn init_tracing() {
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("info,sqlx=warn,hyper=warn"));
|
||||
fmt()
|
||||
.with_env_filter(filter)
|
||||
.with_target(false)
|
||||
.with_thread_ids(true)
|
||||
.with_line_number(true)
|
||||
.compact()
|
||||
.init();
|
||||
}
|
||||
|
||||
#[instrument(skip(db), fields(user_id = %user.id))]
|
||||
async fn process_user(db: &Pool, user: &User) -> anyhow::Result<()> {
|
||||
info!("processing user");
|
||||
if user.is_banned() {
|
||||
warn!(reason = "banned", "skipping");
|
||||
return Ok(());
|
||||
}
|
||||
// ... body ...
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
Replace `println!` with `info!`/`warn!`/`error!`. Replace `eprintln!` with `tracing::error!`.
|
||||
|
||||
## Error reporting (binaries) — `color-eyre`
|
||||
|
||||
For binary `main()`, hook `color-eyre` to give pretty panics + nice `Result` printing:
|
||||
|
||||
```rust
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
tracing_subscriber::fmt::init();
|
||||
real_main()
|
||||
}
|
||||
```
|
||||
|
||||
Library code stays on `anyhow`/`thiserror`. `color-eyre` is purely a display layer for the binary.
|
||||
|
||||
## Serialization — `serde` + `serde_json`
|
||||
|
||||
The default for any data crossing a process boundary (file, network, IPC, database column).
|
||||
|
||||
```rust
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields, rename_all = "snake_case")]
|
||||
pub struct ApiResponse {
|
||||
pub user_id: UserId,
|
||||
pub created_at: jiff::Timestamp,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub display_name: Option<String>,
|
||||
#[serde(flatten)]
|
||||
pub extra: HashMap<String, serde_json::Value>,
|
||||
}
|
||||
```
|
||||
|
||||
`deny_unknown_fields` catches typos in inputs. `rename_all = "snake_case"` aligns with REST/JSON conventions while keeping idiomatic Rust field names. `#[serde(flatten)]` for forward-compatible extra fields.
|
||||
|
||||
Alternatives:
|
||||
- `serde_yaml` (YAML — note: YAML's "deserialize anything" surface is a security trap; prefer JSON/TOML where possible)
|
||||
- `toml` (config files)
|
||||
- `rmp-serde` (MessagePack — binary, fast)
|
||||
- `ciborium` (CBOR)
|
||||
- `bincode 2` (binary, smaller; no serde required in v2 but interop fine)
|
||||
|
||||
## HTTP client — `reqwest`
|
||||
|
||||
```rust
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(30))
|
||||
.user_agent(concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")))
|
||||
.https_only(true)
|
||||
.pool_max_idle_per_host(8)
|
||||
.build()?;
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Repo { full_name: String, stargazers_count: u64 }
|
||||
|
||||
let repo: Repo = client
|
||||
.get("https://api.github.com/repos/rust-lang/rust")
|
||||
.send().await?
|
||||
.error_for_status()?
|
||||
.json().await?;
|
||||
```
|
||||
|
||||
`error_for_status()?` turns 4xx/5xx into `Err`. Always include a User-Agent. `https_only(true)` is a soundness toggle - prevents accidental http:// downgrade.
|
||||
|
||||
## Web framework — `axum`
|
||||
|
||||
```rust
|
||||
use axum::{Router, routing::get, extract::State, response::Json};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct AppState { db: sqlx::PgPool }
|
||||
|
||||
async fn health(State(state): State<Arc<AppState>>) -> Json<serde_json::Value> {
|
||||
let ok = sqlx::query_scalar!("SELECT 1::int4").fetch_one(&state.db).await.is_ok();
|
||||
Json(serde_json::json!({ "ok": ok }))
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
tracing_subscriber::fmt::init();
|
||||
let state = Arc::new(AppState { db: sqlx::PgPool::connect(&env_db()).await? });
|
||||
let app = Router::new()
|
||||
.route("/health", get(health))
|
||||
.with_state(state)
|
||||
.layer(tower_http::trace::TraceLayer::new_for_http())
|
||||
.layer(tower_http::compression::CompressionLayer::new());
|
||||
let listener = tokio::net::TcpListener::bind("0.0.0.0:8080").await?;
|
||||
axum::serve(listener, app).await?;
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
Avoid `actix-web` (legacy patterns, separate runtime model), `warp` (filter explosion in non-trivial apps), `rocket` (slow release cadence). Pair `axum` with `tower-http` for middleware (trace, compression, CORS, timeout, request-id).
|
||||
|
||||
## Database — `sqlx` (compile-time checked SQL)
|
||||
|
||||
```rust
|
||||
use sqlx::PgPool;
|
||||
|
||||
#[derive(Debug, sqlx::FromRow)]
|
||||
pub struct User { pub id: uuid::Uuid, pub email: String, pub created_at: jiff::Timestamp }
|
||||
|
||||
pub async fn find_user(pool: &PgPool, email: &str) -> Result<Option<User>, sqlx::Error> {
|
||||
sqlx::query_as!(
|
||||
User,
|
||||
r#"SELECT id, email, created_at as "created_at: jiff::Timestamp"
|
||||
FROM users WHERE email = $1"#,
|
||||
email
|
||||
)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
}
|
||||
```
|
||||
|
||||
`query_as!` checks the SQL against the live database at compile time. To work without a live DB during builds, generate offline metadata: `cargo sqlx prepare`. Commit the resulting `.sqlx/` directory.
|
||||
|
||||
Avoid `diesel` (sync-first, heavy DSL), raw `tokio-postgres` (loses type checks), `sea-orm` (more magic, less control).
|
||||
|
||||
For migrations: `sqlx migrate add <name>` + `sqlx::migrate!("./migrations").run(&pool).await?`.
|
||||
|
||||
## Time — `jiff`
|
||||
|
||||
The 2025+ choice. Single crate, sane defaults, civil time / instant / span distinction.
|
||||
|
||||
```rust
|
||||
use jiff::{Timestamp, Span, ToSpan, Zoned};
|
||||
|
||||
let now: Timestamp = Timestamp::now();
|
||||
let in_one_hour = now.checked_add(1.hour())?;
|
||||
let local: Zoned = now.in_tz("Asia/Seoul")?;
|
||||
let span: Span = local - some_earlier.in_tz("Asia/Seoul")?;
|
||||
```
|
||||
|
||||
Avoid `chrono` (old API, generic-heavy, time zone story still painful), `time` crate (split ecosystem, weaker docs). `jiff` is the post-`chrono` consolidation.
|
||||
|
||||
## UUID — `uuid` with v7
|
||||
|
||||
```rust
|
||||
use uuid::Uuid;
|
||||
|
||||
// v7 for IDs (sortable, time-ordered, monotonic-ish, RFC 9562)
|
||||
let id = Uuid::now_v7();
|
||||
```
|
||||
|
||||
v4 is fine for nonces, v7 for primary keys (better index locality). Never v1 (leaks MAC). Cargo features: `uuid = { version = "1", features = ["v4", "v7", "serde"] }`.
|
||||
|
||||
## DataFrames / analytics — `polars`
|
||||
|
||||
For columnar data, joins, group-by, lazy plans:
|
||||
|
||||
```rust
|
||||
use polars::prelude::*;
|
||||
|
||||
let df = LazyCsvReader::new("events.csv")
|
||||
.finish()?
|
||||
.group_by([col("user_id")])
|
||||
.agg([col("amount").sum().alias("total")])
|
||||
.sort(["total"], Default::default())
|
||||
.collect()?;
|
||||
```
|
||||
|
||||
The Rust API mirrors the Python one. Use the lazy API by default; materialize with `.collect()` at the end.
|
||||
|
||||
## Channels
|
||||
|
||||
- Single-producer single-consumer or bounded MPSC → `tokio::sync::mpsc` (async) or `flume` (sync + async).
|
||||
- Broadcast → `tokio::sync::broadcast`.
|
||||
- Watch (latest-value pubsub) → `tokio::sync::watch`.
|
||||
- Oneshot → `tokio::sync::oneshot`.
|
||||
|
||||
Avoid raw `std::sync::mpsc` (sync only, fewer features), `crossbeam-channel` (good but heavier; use only if you need rendezvous semantics).
|
||||
|
||||
## Coordinate spaces / 2D math — `euclid`
|
||||
|
||||
```rust
|
||||
use euclid::{Point2D, Size2D, default::Box2D};
|
||||
struct ScreenSpace;
|
||||
struct WorldSpace;
|
||||
|
||||
type ScreenPoint = Point2D<f32, ScreenSpace>;
|
||||
type WorldPoint = Point2D<f32, WorldSpace>;
|
||||
|
||||
let cursor: ScreenPoint = Point2D::new(120.0, 240.0);
|
||||
let player: WorldPoint = Point2D::new(3.5, 1.2);
|
||||
|
||||
// let mistake = cursor + player; // ❌ type error
|
||||
```
|
||||
|
||||
Generalize the pattern to your own domains (see `references/type-state.md`).
|
||||
|
||||
## Property tests — `proptest`
|
||||
|
||||
```rust
|
||||
use proptest::prelude::*;
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn parse_roundtrips(s in r"[a-zA-Z0-9_-]{1,50}") {
|
||||
let parsed = parse(&s).unwrap();
|
||||
let back = parsed.to_string();
|
||||
prop_assert_eq!(back, s);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Avoid `quickcheck` (older, less ergonomic). proptest gives shrinking + regression corpus + integration with `criterion`.
|
||||
|
||||
## Snapshot tests — `insta`
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
fn renders_help() {
|
||||
let output = render(&example_input());
|
||||
insta::assert_snapshot!(output);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serializes_well() {
|
||||
insta::assert_json_snapshot!(serializable_value());
|
||||
}
|
||||
```
|
||||
|
||||
`cargo insta review` (after `cargo install cargo-insta`) — interactive review of changed snapshots.
|
||||
|
||||
## Benchmarks — `criterion`
|
||||
|
||||
Stable Rust friendly (no nightly `#[bench]`).
|
||||
|
||||
```rust
|
||||
use criterion::{black_box, criterion_group, criterion_main, Criterion};
|
||||
|
||||
fn bench_parse(c: &mut Criterion) {
|
||||
let input = std::fs::read_to_string("samples/large.txt").unwrap();
|
||||
c.bench_function("parse_large", |b| b.iter(|| parse(black_box(&input))));
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_parse);
|
||||
criterion_main!(benches);
|
||||
```
|
||||
|
||||
Run with `cargo bench`. HTML reports under `target/criterion/`. Pair with `cargo bench -- --save-baseline main` then `--baseline main` for comparison.
|
||||
|
||||
## Concurrency model — `loom`
|
||||
|
||||
For lock-free or atomic-heavy code (channels, refcounts, hazard pointers). See `references/concurrency.md` for the full pattern.
|
||||
|
||||
## Arena allocator — `bumpalo`
|
||||
|
||||
```rust
|
||||
use bumpalo::Bump;
|
||||
|
||||
let bump = Bump::new();
|
||||
let node = bump.alloc(Node { value: 42, next: None });
|
||||
let s: &str = bump.alloc_str("hello");
|
||||
// All allocations freed at once when `bump` drops.
|
||||
```
|
||||
|
||||
For parser nodes, AST construction, per-request scratch. Outperforms heap allocation for short-lived owned data by an order of magnitude.
|
||||
|
||||
## Web client (browser, WASM-bound) — `gloo` ecosystem
|
||||
|
||||
If targeting WASM browser, use `gloo-net` for fetch and `gloo-storage` for localStorage; not `web-sys` directly unless you need DOM-level APIs.
|
||||
|
||||
## Lazy statics — `std::sync::LazyLock` (since 1.80)
|
||||
|
||||
```rust
|
||||
use std::sync::LazyLock;
|
||||
static CONFIG: LazyLock<Config> = LazyLock::new(|| Config::load_from_env().unwrap());
|
||||
```
|
||||
|
||||
Avoid `lazy_static!` (macro-heavy, predates std), `once_cell` (now in std as `LazyLock`/`OnceLock`).
|
||||
|
||||
## Hash maps — `std::collections::HashMap` + `ahash` for hot paths
|
||||
|
||||
```rust
|
||||
use std::collections::HashMap;
|
||||
use ahash::RandomState;
|
||||
|
||||
type FastMap<K, V> = HashMap<K, V, RandomState>;
|
||||
let mut counters: FastMap<String, u64> = FastMap::default();
|
||||
```
|
||||
|
||||
`HashMap` defaults to SipHash (DoS-resistant). For internal hot loops where you trust the keys, `ahash` is 2-5x faster.
|
||||
|
||||
For sorted iteration, use `BTreeMap`. For small keys with known small N, `Vec<(K, V)>` may beat both.
|
||||
|
||||
## File I/O — `tokio::fs` (async) or `std::fs` (sync utility)
|
||||
|
||||
```rust
|
||||
let contents = tokio::fs::read_to_string("data.json").await?;
|
||||
```
|
||||
|
||||
For large files: `tokio::fs::File` + `tokio::io::BufReader`. For random access, `memmap2` (with the unsafe-discipline wrappers).
|
||||
|
||||
## Decision tree
|
||||
|
||||
```
|
||||
Need to ship the thing?
|
||||
├── HTTP server → axum + sqlx + tracing + jiff + tokio
|
||||
├── HTTP client → reqwest (+ tokio)
|
||||
├── CLI → clap + color-eyre + tracing + indicatif (progress) + dialoguer (prompts)
|
||||
├── TUI → ratatui + crossterm
|
||||
├── Background worker → tokio + ETL → polars + duckdb
|
||||
├── Game / graphics → wgpu + winit + euclid (or bevy if you want the engine)
|
||||
├── WASM front-end → leptos (or dioxus / yew) + wasm-bindgen + gloo
|
||||
├── Embedded → embassy (async on bare metal)
|
||||
├── FFI to C / Python → cxx (C++) / pyo3 (Python) / cbindgen (header gen)
|
||||
└── Just a script → rust-script (see one-liners.md)
|
||||
```
|
||||
|
||||
When in doubt, search crates.io for the latest version, then check:
|
||||
1. Is it maintained? (`cargo deny check` will scream if it's yanked or unmaintained)
|
||||
2. Does it have `serde` feature? (boundary types should always serde)
|
||||
3. Does it have `tokio` integration? (avoid runtime mixing)
|
||||
4. Is it on `tokio::io::AsyncRead`/`AsyncWrite` (the std for async I/O)?
|
||||
5. Are there safety-critical `unsafe` regions? If yes, has the author shipped miri proofs?
|
||||
@@ -0,0 +1,291 @@
|
||||
# One-Liners and Disposable Scripts
|
||||
|
||||
Production hygiene with throwaway ergonomics. Rust scripts get the same strict lints, the same miri rule when `unsafe` is touched, the same type discipline. The difference is dependency declaration lives inline.
|
||||
|
||||
## `rust-script` — the recommended path
|
||||
|
||||
Install once:
|
||||
|
||||
```bash
|
||||
cargo install rust-script
|
||||
```
|
||||
|
||||
Write a script:
|
||||
|
||||
```rust
|
||||
#!/usr/bin/env rust-script
|
||||
//! Fetch a URL and print its body length.
|
||||
//!
|
||||
//! Usage:
|
||||
//! ./fetch.rs <url>
|
||||
//!
|
||||
//! ```cargo
|
||||
//! [dependencies]
|
||||
//! anyhow = "1"
|
||||
//! reqwest = { version = "0.12", features = ["blocking"] }
|
||||
//! ```
|
||||
|
||||
use std::env;
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
let url = env::args().nth(1).context("usage: fetch.rs <url>")?;
|
||||
let body = reqwest::blocking::get(&url)?.error_for_status()?.text()?;
|
||||
println!("{} bytes", body.len());
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
Make executable: `chmod +x fetch.rs`. Run: `./fetch.rs https://example.com`.
|
||||
|
||||
The `//! \`\`\`cargo` block is parsed as inline `Cargo.toml`. Everything else is normal Rust.
|
||||
|
||||
## With async
|
||||
|
||||
```rust
|
||||
#!/usr/bin/env rust-script
|
||||
//! ```cargo
|
||||
//! [dependencies]
|
||||
//! anyhow = "1"
|
||||
//! tokio = { version = "1", features = ["full"] }
|
||||
//! reqwest = "0.12"
|
||||
//! ```
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let urls = [
|
||||
"https://example.com",
|
||||
"https://example.org",
|
||||
];
|
||||
let client = reqwest::Client::new();
|
||||
let bodies = futures::future::join_all(urls.iter().map(|u| {
|
||||
let c = client.clone();
|
||||
async move { c.get(*u).send().await?.text().await }
|
||||
})).await;
|
||||
for (url, body) in urls.iter().zip(bodies) {
|
||||
match body {
|
||||
Ok(b) => println!("{url}: {} bytes", b.len()),
|
||||
Err(e) => eprintln!("{url}: error {e}"),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
## With CLI parsing
|
||||
|
||||
```rust
|
||||
#!/usr/bin/env rust-script
|
||||
//! ```cargo
|
||||
//! [dependencies]
|
||||
//! anyhow = "1"
|
||||
//! clap = { version = "4", features = ["derive"] }
|
||||
//! ```
|
||||
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(version, about = "rename files by pattern")]
|
||||
struct Cli {
|
||||
/// Glob to match
|
||||
pattern: String,
|
||||
/// Replacement template (use {n} for sequence)
|
||||
template: String,
|
||||
/// Show what would happen without doing it
|
||||
#[arg(long)]
|
||||
dry_run: bool,
|
||||
}
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
let cli = Cli::parse();
|
||||
let entries: Vec<_> = glob::glob(&cli.pattern)?.collect::<Result<_, _>>()?;
|
||||
for (n, entry) in entries.iter().enumerate() {
|
||||
let target = cli.template.replace("{n}", &n.to_string());
|
||||
if cli.dry_run {
|
||||
println!("{} -> {target}", entry.display());
|
||||
} else {
|
||||
std::fs::rename(entry, &target)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
## Caching
|
||||
|
||||
`rust-script` caches the compiled binary in `~/.cache/rust-script/`. First run is slow (full compile), subsequent runs are instant.
|
||||
|
||||
To clear: `rust-script --clear-cache`.
|
||||
|
||||
Pin a script's compile target into the script directory for portability:
|
||||
|
||||
```bash
|
||||
rust-script --build-only --base-path . ./script.rs
|
||||
```
|
||||
|
||||
This drops a `target/` next to the script with the prebuilt binary.
|
||||
|
||||
## `cargo-script` (RFC 3424, stable since Rust 1.85)
|
||||
|
||||
The official replacement that landed in cargo proper. Same idea, slightly different syntax:
|
||||
|
||||
```rust
|
||||
#!/usr/bin/env -S cargo +nightly -Zscript
|
||||
---
|
||||
package:
|
||||
name = "fetch"
|
||||
edition = "2024"
|
||||
|
||||
dependencies:
|
||||
anyhow = "1"
|
||||
reqwest = { version = "0.12", features = ["blocking"] }
|
||||
---
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
let url = std::env::args().nth(1).context("url required")?;
|
||||
println!("{}", reqwest::blocking::get(&url)?.text()?.len());
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
Status as of 2026-05: stabilization in progress. Use `rust-script` for production now, migrate when `cargo script` is stable everywhere your tools live.
|
||||
|
||||
## Strict mode for scripts
|
||||
|
||||
Add a lints block in the inline `Cargo.toml`:
|
||||
|
||||
```rust
|
||||
//! ```cargo
|
||||
//! [dependencies]
|
||||
//! anyhow = "1"
|
||||
//!
|
||||
//! [lints.rust]
|
||||
//! unsafe_code = "forbid"
|
||||
//!
|
||||
//! [lints.clippy]
|
||||
//! all = "deny"
|
||||
//! pedantic = "warn"
|
||||
//! unwrap_used = "deny"
|
||||
//! expect_used = "deny"
|
||||
//! panic = "deny"
|
||||
//! ```
|
||||
```
|
||||
|
||||
Now the script gets the same strictness as the main project. If you need a one-line `unwrap()` for prototype velocity, switch the lint to `warn` for that one script - never blanket `allow`.
|
||||
|
||||
Run with lints visible:
|
||||
|
||||
```bash
|
||||
RUSTFLAGS="-D warnings" rust-script ./script.rs
|
||||
```
|
||||
|
||||
## When NOT to use a script
|
||||
|
||||
- It is going to live longer than a week → make it a real crate with `cargo new --bin`.
|
||||
- It needs custom build scripts (`build.rs`) → real crate.
|
||||
- It needs binary distribution to other machines → real crate with `cargo dist`.
|
||||
- It needs to be tested → real crate (scripts can technically run `#[test]`s under `cargo test`, but the workflow is awkward).
|
||||
|
||||
A reasonable migration path: start as a script, when complexity grows past ~200 lines or you reach for a second `.rs` file, run `rust-script --emit ./script.rs` to dump a regular Cargo project skeleton and continue from there.
|
||||
|
||||
## Inline tests in a script
|
||||
|
||||
```rust
|
||||
#!/usr/bin/env rust-script
|
||||
//! ```cargo
|
||||
//! [dependencies]
|
||||
//! ```
|
||||
|
||||
fn double(x: i32) -> i32 { x * 2 }
|
||||
|
||||
fn main() {
|
||||
println!("{}", double(21));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn doubles_ints() {
|
||||
assert_eq!(double(5), 10);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Run tests: `rust-script --test ./script.rs`.
|
||||
|
||||
## A useful "Rust as awk" pattern
|
||||
|
||||
For ad-hoc data processing on stdin:
|
||||
|
||||
```rust
|
||||
#!/usr/bin/env rust-script
|
||||
//! ```cargo
|
||||
//! [dependencies]
|
||||
//! serde_json = "1"
|
||||
//! ```
|
||||
|
||||
use std::io::{self, BufRead, Write};
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let stdin = io::stdin();
|
||||
let stdout = io::stdout();
|
||||
let mut out = stdout.lock();
|
||||
for line in stdin.lock().lines() {
|
||||
let line = line?;
|
||||
let value: serde_json::Value = serde_json::from_str(&line)?;
|
||||
if let Some(s) = value.get("level").and_then(|v| v.as_str()) {
|
||||
if s == "error" {
|
||||
writeln!(out, "{line}")?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
`cat logs.jsonl | ./filter-errors.rs` — filter JSON logs by `level == "error"`. Faster than `jq` for big files, type-safe.
|
||||
|
||||
For numerics:
|
||||
|
||||
```rust
|
||||
#!/usr/bin/env rust-script
|
||||
//! sum a column of numbers from stdin
|
||||
use std::io::{self, BufRead};
|
||||
fn main() {
|
||||
let total: f64 = io::stdin().lock().lines()
|
||||
.filter_map(|l| l.ok())
|
||||
.filter_map(|l| l.trim().parse::<f64>().ok())
|
||||
.sum();
|
||||
println!("{total}");
|
||||
}
|
||||
```
|
||||
|
||||
## The `rust-script` shebang trick on macOS
|
||||
|
||||
macOS does not support multi-arg shebangs without `env -S`. Use:
|
||||
|
||||
```rust
|
||||
#!/usr/bin/env -S rust-script --
|
||||
```
|
||||
|
||||
The `--` lets clap-style argument parsers see the user's args, not the rust-script arguments.
|
||||
|
||||
## Editor support
|
||||
|
||||
VS Code / Helix / Vim with `rust-analyzer`: open the script file as if it were `src/main.rs` of an inferred crate. Most editors auto-detect the inline manifest. If not, hand-create a `Cargo.toml` next to the script with matching deps for the duration of editing, then delete it.
|
||||
|
||||
## When `rust-script` is too heavy
|
||||
|
||||
For absolutely throwaway "one expression on stdin" use cases, a Rust REPL like `evcxr_jupyter` (Jupyter kernel) or `irust` (terminal REPL) is more appropriate:
|
||||
|
||||
```bash
|
||||
cargo install irust
|
||||
irust
|
||||
```
|
||||
|
||||
But these are interactive playgrounds, not scriptable. For shell pipelines, stay with `rust-script`.
|
||||
|
||||
## The Promise
|
||||
|
||||
Same strict lints. Same `clippy::pedantic` enforcement. Same `unsafe`-requires-SAFETY rule. The agent does not get a free pass on a 30-line script. The whole point of strict scripts is that **production hygiene is cheap when the toolchain enforces it**.
|
||||
@@ -0,0 +1,429 @@
|
||||
# Property Tests (proptest) + Snapshot Tests (insta)
|
||||
|
||||
Two test types every Rust project should have alongside unit tests. Proptest hunts for inputs your unit tests forgot to try. Insta locks down output shapes you do not want to silently change.
|
||||
|
||||
## When to reach for each
|
||||
|
||||
| Want to test… | Use |
|
||||
|---|---|
|
||||
| One specific behavior with a known input | `#[test]` + `assert_eq!` |
|
||||
| All inputs of a certain shape work | `proptest!` |
|
||||
| Output structure stays stable across refactors | `insta::assert_*_snapshot!` |
|
||||
| Parser/serializer round-trips | `proptest!` (the round-trip property) |
|
||||
| CLI help text, JSON response shape, debug output | `insta::assert_snapshot!` |
|
||||
| Concurrency under all interleavings | `loom` (see `concurrency.md`) |
|
||||
|
||||
Use all three. They cover different bug classes.
|
||||
|
||||
## Proptest — setup
|
||||
|
||||
`Cargo.toml`:
|
||||
|
||||
```toml
|
||||
[dev-dependencies]
|
||||
proptest = "1"
|
||||
proptest-derive = "0.5" # for #[derive(Arbitrary)]
|
||||
```
|
||||
|
||||
`proptest.toml` at project root (optional, sane defaults):
|
||||
|
||||
```toml
|
||||
cases = 256 # number of random inputs per property
|
||||
max_local_rejects = 65536
|
||||
max_global_rejects = 1024
|
||||
max_shrink_iters = 1024
|
||||
max_shrink_time = 60_000 # ms
|
||||
failure_persistence = { source_file = "proptest-regressions/", file_name = "regressions.txt" }
|
||||
verbose = 0
|
||||
```
|
||||
|
||||
`failure_persistence` is the killer feature: every failure is written to a regression file. On the next run, those exact inputs are replayed first, so once a bug is found it never escapes again.
|
||||
|
||||
## Basic property test
|
||||
|
||||
```rust
|
||||
use proptest::prelude::*;
|
||||
|
||||
fn parse(s: &str) -> Result<Color, ParseError> { /* ... */ }
|
||||
fn render(c: &Color) -> String { /* ... */ }
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn parse_render_roundtrips(red in 0u8..=255, green in 0u8..=255, blue in 0u8..=255) {
|
||||
let color = Color { red, green, blue };
|
||||
let rendered = render(&color);
|
||||
let parsed = parse(&rendered).expect("our render should always parse");
|
||||
prop_assert_eq!(parsed, color);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`proptest!` macro takes `(arg in strategy, ...)` pairs. Each strategy produces values; proptest runs the body with random samples, then shrinks failing cases to minimal forms.
|
||||
|
||||
## Strategies — the value-generation language
|
||||
|
||||
| Strategy | Produces |
|
||||
|---|---|
|
||||
| `any::<T>()` | Any value of `T` (if `T: Arbitrary`) |
|
||||
| `0u32..100` | Integer ranges |
|
||||
| `prop::sample::select(slice)` | Pick from a list |
|
||||
| `prop::collection::vec(elem, range)` | Vec of length in range |
|
||||
| `prop::collection::hash_map(k, v, n..m)` | HashMap |
|
||||
| `prop::option::of(strategy)` | Option |
|
||||
| `prop::result::maybe_ok(ok, err)` | Result |
|
||||
| `(s1, s2).prop_map(\|(a, b)\| ...)` | Combine, transform |
|
||||
| `s.prop_filter("reason", \|v\| pred)` | Reject values |
|
||||
| `s.prop_flat_map(\|v\| dependent)` | Sequential dependency |
|
||||
| `prop_oneof![strategy1, strategy2]` | Union of strategies |
|
||||
| `r"[a-z]{3,10}"` | Regex-generated string |
|
||||
| `"\\PC*"` | Any printable non-control string |
|
||||
|
||||
Example combining several:
|
||||
|
||||
```rust
|
||||
fn config_strategy() -> impl Strategy<Value = Config> {
|
||||
(
|
||||
prop::sample::select(vec!["dev", "staging", "prod"]),
|
||||
0u16..=65535,
|
||||
prop::collection::hash_map(
|
||||
r"[a-z_]{1,20}",
|
||||
any::<String>(),
|
||||
0..5,
|
||||
),
|
||||
).prop_map(|(env, port, vars)| Config {
|
||||
env: env.into(),
|
||||
port,
|
||||
env_vars: vars,
|
||||
})
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn config_validates(cfg in config_strategy()) {
|
||||
let result = validate(&cfg);
|
||||
if cfg.port == 0 {
|
||||
prop_assert!(result.is_err());
|
||||
} else {
|
||||
prop_assert!(result.is_ok());
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Properties to write for every parser
|
||||
|
||||
1. **Round-trip:** `parse(render(x)) == x` for all valid `x`.
|
||||
2. **No-panic:** `parse(arbitrary_string)` never panics, always returns `Result`.
|
||||
3. **Idempotent:** `parse(parse(x).unwrap().render()) == parse(x).unwrap()`.
|
||||
4. **Whitespace insensitivity:** `parse(x) == parse(strip_whitespace(x))` (if applicable).
|
||||
|
||||
For every serializer:
|
||||
|
||||
1. **Length bound:** `render(x).len() <= bound(x)`.
|
||||
2. **Charset:** `render(x).chars().all(|c| ALLOWED.contains(&c))`.
|
||||
|
||||
For every collection operation:
|
||||
|
||||
1. **Identity:** `op_identity(x) == x` (sort an already-sorted, dedupe a unique).
|
||||
2. **Idempotence:** `op(op(x)) == op(x)`.
|
||||
3. **Commutativity:** `op(a, b) == op(b, a)` (set union, etc).
|
||||
4. **Length:** `op(a, b).len() == known_relation(a.len(), b.len())`.
|
||||
|
||||
For every numeric op:
|
||||
|
||||
1. **Monotonicity:** `a <= b => f(a) <= f(b)`.
|
||||
2. **Identity element:** `f(x, identity) == x`.
|
||||
|
||||
Write these mechanically. The agent should reach for proptest the moment any of these properties is checkable.
|
||||
|
||||
## Derive `Arbitrary`
|
||||
|
||||
```rust
|
||||
use proptest_derive::Arbitrary;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Arbitrary)]
|
||||
struct Vec3 {
|
||||
#[proptest(strategy = "-100.0..=100.0")]
|
||||
x: f32,
|
||||
#[proptest(strategy = "-100.0..=100.0")]
|
||||
y: f32,
|
||||
#[proptest(strategy = "-100.0..=100.0")]
|
||||
z: f32,
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn dot_product_is_commutative(a: Vec3, b: Vec3) {
|
||||
prop_assert!((dot(&a, &b) - dot(&b, &a)).abs() < 1e-5);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`#[derive(Arbitrary)]` auto-implements the strategy. Per-field `#[proptest(strategy = "...")]` overrides.
|
||||
|
||||
## Stateful / state machine tests
|
||||
|
||||
For data structures with operations (queues, maps, trees), use `proptest-state-machine`:
|
||||
|
||||
```rust
|
||||
use proptest_state_machine::{ReferenceStateMachine, StateMachineTest};
|
||||
|
||||
struct MyQueueRef { state: VecDeque<i32> }
|
||||
struct MyQueueSut { sut: MyQueue<i32> }
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
enum Op { Push(i32), Pop }
|
||||
|
||||
impl ReferenceStateMachine for MyQueueRef {
|
||||
type State = VecDeque<i32>;
|
||||
type Transition = Op;
|
||||
|
||||
fn init_state() -> BoxedStrategy<Self::State> {
|
||||
Just(VecDeque::new()).boxed()
|
||||
}
|
||||
fn transitions(_: &Self::State) -> BoxedStrategy<Self::Transition> {
|
||||
prop_oneof![
|
||||
any::<i32>().prop_map(Op::Push),
|
||||
Just(Op::Pop),
|
||||
].boxed()
|
||||
}
|
||||
fn apply(mut state: Self::State, transition: &Self::Transition) -> Self::State {
|
||||
match transition {
|
||||
Op::Push(x) => state.push_back(*x),
|
||||
Op::Pop => { state.pop_front(); }
|
||||
}
|
||||
state
|
||||
}
|
||||
}
|
||||
|
||||
impl StateMachineTest for MyQueueSut {
|
||||
type SystemUnderTest = MyQueue<i32>;
|
||||
type Reference = MyQueueRef;
|
||||
|
||||
fn init_test(_: &<Self::Reference as ReferenceStateMachine>::State) -> Self::SystemUnderTest {
|
||||
MyQueue::new()
|
||||
}
|
||||
fn apply(mut sut: Self::SystemUnderTest, _: &VecDeque<i32>, transition: Op) -> Self::SystemUnderTest {
|
||||
match transition {
|
||||
Op::Push(x) => sut.push(x),
|
||||
Op::Pop => { sut.pop(); }
|
||||
}
|
||||
sut
|
||||
}
|
||||
fn check_invariants(sut: &Self::SystemUnderTest, state: &VecDeque<i32>) {
|
||||
assert_eq!(sut.len(), state.len());
|
||||
// also check head/tail/iteration order...
|
||||
}
|
||||
}
|
||||
|
||||
proptest_state_machine::prop_state_machine! {
|
||||
#[test]
|
||||
fn queue_matches_vecdeque(sequential 1..50 => MyQueueSut);
|
||||
}
|
||||
```
|
||||
|
||||
You define a reference implementation (`VecDeque` here), proptest fuzzes operations against both, asserts invariants every step. This is the technique for finding bugs in lock-free or complex containers.
|
||||
|
||||
## Shrinking
|
||||
|
||||
When a property fails, proptest reduces the input to a minimal counter-example. For built-in strategies this is automatic. For custom strategies built with `prop_map`, shrinking goes through the underlying strategy. Avoid breaking shrinking with `prop_filter` (rejection sampling) over wide spaces; prefer `prop_flat_map` or directly-shaped strategies.
|
||||
|
||||
## Regression corpus
|
||||
|
||||
When a property test fails, proptest writes the failing input to `proptest-regressions/<test_name>.txt`. Commit this directory. Future runs replay these failing inputs first, so the bug stays fixed forever.
|
||||
|
||||
```
|
||||
proptest-regressions/
|
||||
└── parse_color.txt # commit this
|
||||
```
|
||||
|
||||
## Insta — setup
|
||||
|
||||
`Cargo.toml`:
|
||||
|
||||
```toml
|
||||
[dev-dependencies]
|
||||
insta = { version = "1", features = ["yaml", "json", "redactions", "filters"] }
|
||||
|
||||
[dependencies.serde_yaml]
|
||||
version = "0.9"
|
||||
optional = true
|
||||
```
|
||||
|
||||
Install the CLI:
|
||||
|
||||
```bash
|
||||
cargo install cargo-insta
|
||||
```
|
||||
|
||||
## Insta — basic snapshots
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
fn renders_default_help() {
|
||||
let output = render_help();
|
||||
insta::assert_snapshot!(output);
|
||||
}
|
||||
```
|
||||
|
||||
First run: creates `src/snapshots/mycrate__renders_default_help.snap.new`. Run `cargo insta review`, press `a` to accept, the `.new` extension is dropped. Subsequent runs diff against the committed snapshot; mismatches fail the test.
|
||||
|
||||
## Insta — typed snapshots
|
||||
|
||||
```rust
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
struct Result {
|
||||
status: String,
|
||||
user: User,
|
||||
duration_ms: u64,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_response() {
|
||||
let value = compute();
|
||||
insta::assert_json_snapshot!(value);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn yaml_response() {
|
||||
insta::assert_yaml_snapshot!(value);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn debug_repr() {
|
||||
insta::assert_debug_snapshot!(value);
|
||||
}
|
||||
```
|
||||
|
||||
Choose:
|
||||
- `assert_snapshot!` for `String`/`Display` output (CLI help, error messages, generated code).
|
||||
- `assert_debug_snapshot!` for `{:?}` (Rust-internal data).
|
||||
- `assert_json_snapshot!` for structured data crossing process boundaries.
|
||||
- `assert_yaml_snapshot!` when YAML is easier to read in diffs.
|
||||
|
||||
## Insta — redactions and filters
|
||||
|
||||
For values that change every run (timestamps, UUIDs, paths):
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
fn with_redactions() {
|
||||
let value = ApiResponse {
|
||||
id: uuid::Uuid::now_v7(),
|
||||
created_at: jiff::Timestamp::now(),
|
||||
body: "hello".into(),
|
||||
};
|
||||
insta::assert_json_snapshot!(value, {
|
||||
".id" => "[uuid]",
|
||||
".created_at" => "[timestamp]",
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
For regex filters applied to all snapshots in a test:
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
fn with_filters() {
|
||||
let mut settings = insta::Settings::clone_current();
|
||||
settings.add_filter(r"/tmp/[a-z0-9-]+", "[TMP]");
|
||||
settings.add_filter(r"\d+\.\d+ms", "[TIMING]");
|
||||
settings.bind(|| {
|
||||
let output = run_command();
|
||||
insta::assert_snapshot!(output);
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
`Settings::bind` scopes filters to the closure.
|
||||
|
||||
## Insta workflow
|
||||
|
||||
1. Write the test, run it. First run creates `.snap.new`.
|
||||
2. `cargo insta review` → interactive UI. Show diff, accept/reject.
|
||||
3. Accepted snapshots commit to the repo.
|
||||
4. Refactor code. Tests run; mismatches show as diffs.
|
||||
5. If the new output is correct, `cargo insta accept` (or selective `review`). If wrong, fix the code.
|
||||
|
||||
Pair with CI to fail builds when uncommitted `.snap.new` files exist:
|
||||
|
||||
```bash
|
||||
cargo nextest run
|
||||
if find . -name "*.snap.new" | grep -q .; then
|
||||
echo "Pending snapshots, run 'cargo insta review'"
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
## Inline snapshots
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
fn small_output() {
|
||||
let value = compute();
|
||||
insta::assert_snapshot!(value, @"hello world");
|
||||
}
|
||||
```
|
||||
|
||||
The trailing `@"..."` string is the expected snapshot, stored in source. Useful when the value is short enough that pulling out a separate file is overkill. `cargo insta accept` updates them in-place.
|
||||
|
||||
## Inline JSON snapshots
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
fn json_inline() {
|
||||
insta::assert_json_snapshot!(value, @r###"
|
||||
{
|
||||
"status": "ok",
|
||||
"count": 3
|
||||
}
|
||||
"###);
|
||||
}
|
||||
```
|
||||
|
||||
## Anti-patterns
|
||||
|
||||
1. **Snapshots of unstable output.** If `HashMap` iteration order changes per run, snapshots will fail. Switch to `BTreeMap` or sort before snapshotting.
|
||||
2. **Massive snapshots.** A 10KB JSON dump where you really care about 3 fields. Either narrow to the fields, or accept that any refactor will require re-reviewing 10KB.
|
||||
3. **Snapshots that bake in implementation details.** "function called 3 times" is not a snapshot - it's a behavior assertion. Use a real assertion.
|
||||
4. **Skipping `cargo insta review`.** Accepting blind via `cargo insta accept --all` defeats the purpose. Always review.
|
||||
|
||||
## Combining proptest + insta
|
||||
|
||||
```rust
|
||||
proptest! {
|
||||
#[test]
|
||||
fn random_inputs_render_consistently(input: ValidInput) {
|
||||
let mut settings = insta::Settings::clone_current();
|
||||
settings.set_snapshot_suffix(format!("{}", input.hash()));
|
||||
settings.bind(|| {
|
||||
insta::assert_snapshot!(render(&input));
|
||||
});
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
But honestly, this is rarely a fit. Proptest tests properties, insta tests output shape. Don't snapshot random inputs - that defeats both tools.
|
||||
|
||||
## CI matrix recommendation
|
||||
|
||||
```yaml
|
||||
- name: Tests
|
||||
run: cargo nextest run --all-features
|
||||
|
||||
- name: Property regressions (replay)
|
||||
run: |
|
||||
# The regression files in proptest-regressions/ replay first.
|
||||
# Failures here mean a previously-fixed bug came back.
|
||||
cargo nextest run --all-features --test-threads 1
|
||||
```
|
||||
|
||||
When a proptest finds a new failure, the regression file appears as a git diff - check it in.
|
||||
|
||||
## What proptest cannot do
|
||||
|
||||
- Find bugs that require multi-process / multi-network coordination → integration tests + fault injection.
|
||||
- Find concurrency bugs → use `loom` (see `concurrency.md`).
|
||||
- Find performance regressions → use `criterion`.
|
||||
|
||||
But for any function with a domain (inputs to outputs), proptest can find more bugs than your unit tests. **Write the property first, derive the unit test second.**
|
||||
@@ -0,0 +1,354 @@
|
||||
# Type-State and Newtype Patterns
|
||||
|
||||
The single highest-leverage thing Rust gives a coding agent: encode invariants in the type system so the compiler refuses incorrect code. The agent does not have to "remember" rules - the rules are physical.
|
||||
|
||||
## The Two Core Patterns
|
||||
|
||||
1. **Newtype wrappers for distinct semantic units.** Money, IDs, byte offsets, coordinate spaces - each gets its own tuple struct. The agent cannot pass meters where feet are expected, even though both are `f64` under the hood. This is the `euclid::Point<Screen>` vs `euclid::Point<World>` example Chris Allen called out.
|
||||
2. **Type-state for state machines.** Instead of a struct with a `status: enum { Draft, Validated, Persisted }` field and methods that check `if self.status == ...`, model each state as its own type. Transitions are method calls that consume `self` and return a new type. Illegal transitions become unrepresentable.
|
||||
|
||||
## Newtype Wrapper Cookbook
|
||||
|
||||
### Domain IDs
|
||||
|
||||
```rust
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct UserId(Uuid);
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct ProductId(Uuid);
|
||||
|
||||
impl UserId {
|
||||
pub fn new() -> Self { Self(Uuid::now_v7()) }
|
||||
pub fn as_uuid(&self) -> &Uuid { &self.0 }
|
||||
}
|
||||
|
||||
impl ProductId {
|
||||
pub fn new() -> Self { Self(Uuid::now_v7()) }
|
||||
pub fn as_uuid(&self) -> &Uuid { &self.0 }
|
||||
}
|
||||
|
||||
// `fn buy(user: UserId, product: ProductId)` cannot be called with arguments swapped.
|
||||
```
|
||||
|
||||
`#[serde(transparent)]` keeps JSON/SQL round-trips identical to a bare `Uuid` - the wrapper is purely a compile-time discipline.
|
||||
|
||||
### Quantities with Phantom Type Tags
|
||||
|
||||
```rust
|
||||
use core::marker::PhantomData;
|
||||
use core::ops::{Add, Sub, Mul};
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct Quantity<Unit> {
|
||||
raw: f64,
|
||||
_unit: PhantomData<Unit>,
|
||||
}
|
||||
|
||||
// Tag types - zero-sized, never instantiated.
|
||||
pub struct Meters;
|
||||
pub struct Feet;
|
||||
pub struct Seconds;
|
||||
|
||||
impl<U> Quantity<U> {
|
||||
pub const fn new(value: f64) -> Self {
|
||||
Self { raw: value, _unit: PhantomData }
|
||||
}
|
||||
pub fn raw(self) -> f64 { self.raw }
|
||||
}
|
||||
|
||||
// Adding same-unit quantities: allowed.
|
||||
impl<U> Add for Quantity<U> {
|
||||
type Output = Self;
|
||||
fn add(self, rhs: Self) -> Self { Self::new(self.raw + rhs.raw) }
|
||||
}
|
||||
|
||||
// Subtraction: allowed.
|
||||
impl<U> Sub for Quantity<U> {
|
||||
type Output = Self;
|
||||
fn sub(self, rhs: Self) -> Self { Self::new(self.raw - rhs.raw) }
|
||||
}
|
||||
|
||||
// Multiplying by scalar: allowed.
|
||||
impl<U> Mul<f64> for Quantity<U> {
|
||||
type Output = Self;
|
||||
fn mul(self, rhs: f64) -> Self { Self::new(self.raw * rhs) }
|
||||
}
|
||||
|
||||
// Conversions are explicit, named methods - never `From`/`Into` between units.
|
||||
impl Quantity<Meters> {
|
||||
pub fn to_feet(self) -> Quantity<Feet> {
|
||||
Quantity::new(self.raw * 3.280_84)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Now:
|
||||
|
||||
```rust
|
||||
let distance: Quantity<Meters> = Quantity::new(100.0);
|
||||
let height: Quantity<Feet> = Quantity::new(50.0);
|
||||
let combined = distance + height; // ❌ compile error
|
||||
let combined = distance + height.to_feet().to_meters_oops(); // ❌ no such method
|
||||
let combined = distance + distance; // ✅
|
||||
```
|
||||
|
||||
The agent cannot accidentally mix units. Refactors that change a quantity's underlying unit are caught at compile time everywhere the type flows.
|
||||
|
||||
### Byte Offsets vs Character Offsets
|
||||
|
||||
```rust
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct ByteOffset(pub u32);
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct CharOffset(pub u32);
|
||||
|
||||
impl ByteOffset {
|
||||
pub fn add(self, delta: u32) -> Self { Self(self.0 + delta) }
|
||||
}
|
||||
|
||||
// Converting between them is a function on the actual text.
|
||||
pub fn byte_to_char(text: &str, byte: ByteOffset) -> Option<CharOffset> {
|
||||
text.get(..byte.0 as usize).map(|prefix| CharOffset(prefix.chars().count() as u32))
|
||||
}
|
||||
```
|
||||
|
||||
A function signature `fn slice(text: &str, start: ByteOffset, end: ByteOffset)` cannot be called with character offsets. The UTF-8 boundary bug is now a compile error.
|
||||
|
||||
### Currency
|
||||
|
||||
```rust
|
||||
use rust_decimal::Decimal;
|
||||
|
||||
pub struct Krw;
|
||||
pub struct Usd;
|
||||
pub struct Jpy;
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct Money<Currency> {
|
||||
amount: Decimal,
|
||||
_ccy: PhantomData<Currency>,
|
||||
}
|
||||
|
||||
impl<C> Money<C> {
|
||||
pub const fn new(amount: Decimal) -> Self { Self { amount, _ccy: PhantomData } }
|
||||
}
|
||||
|
||||
impl<C> Add for Money<C> {
|
||||
type Output = Self;
|
||||
fn add(self, rhs: Self) -> Self { Self::new(self.amount + rhs.amount) }
|
||||
}
|
||||
|
||||
// No blanket From<Money<X>> for Money<Y> - conversions go through an explicit
|
||||
// FX rate function that takes a `Rate<From, To>` argument.
|
||||
|
||||
pub struct Rate<From, To> {
|
||||
factor: Decimal,
|
||||
_from: PhantomData<From>,
|
||||
_to: PhantomData<To>,
|
||||
}
|
||||
|
||||
impl<From, To> Money<From> {
|
||||
pub fn convert(self, rate: Rate<From, To>) -> Money<To> {
|
||||
Money::new(self.amount * rate.factor)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The agent cannot add KRW and USD by accident. They cannot convert without a rate. They cannot apply a USD→JPY rate to a KRW value.
|
||||
|
||||
### Paths Rooted at Different Bases
|
||||
|
||||
```rust
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// A path guaranteed to be relative to the project root.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ProjectRel(PathBuf);
|
||||
|
||||
/// A path guaranteed to be relative to the user's home directory.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HomeRel(PathBuf);
|
||||
|
||||
impl ProjectRel {
|
||||
pub fn new(path: impl AsRef<Path>) -> Result<Self, PathError> {
|
||||
let path = path.as_ref();
|
||||
if path.is_absolute() { return Err(PathError::NotRelative); }
|
||||
if path.components().any(|c| matches!(c, std::path::Component::ParentDir)) {
|
||||
return Err(PathError::EscapesRoot);
|
||||
}
|
||||
Ok(Self(path.to_path_buf()))
|
||||
}
|
||||
|
||||
pub fn resolve(&self, project_root: &Path) -> PathBuf {
|
||||
project_root.join(&self.0)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The agent's path-handling code now distinguishes between project-relative and home-relative paths at the type level. A function taking `ProjectRel` cannot be called with a `HomeRel`.
|
||||
|
||||
## Type-State State Machines
|
||||
|
||||
Encode the lifecycle of a value as a sequence of types. Each transition consumes the previous state and returns the next.
|
||||
|
||||
### HTTP Request Builder
|
||||
|
||||
```rust
|
||||
pub struct RequestBuilder<State> {
|
||||
url: String,
|
||||
method: Method,
|
||||
headers: HeaderMap,
|
||||
body: Option<Vec<u8>>,
|
||||
_state: PhantomData<State>,
|
||||
}
|
||||
|
||||
pub struct NeedsUrl;
|
||||
pub struct NeedsMethod;
|
||||
pub struct Ready;
|
||||
|
||||
impl RequestBuilder<NeedsUrl> {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
url: String::new(),
|
||||
method: Method::GET,
|
||||
headers: HeaderMap::new(),
|
||||
body: None,
|
||||
_state: PhantomData,
|
||||
}
|
||||
}
|
||||
pub fn url(mut self, url: impl Into<String>) -> RequestBuilder<NeedsMethod> {
|
||||
self.url = url.into();
|
||||
RequestBuilder { url: self.url, method: self.method, headers: self.headers, body: self.body, _state: PhantomData }
|
||||
}
|
||||
}
|
||||
|
||||
impl RequestBuilder<NeedsMethod> {
|
||||
pub fn method(mut self, method: Method) -> RequestBuilder<Ready> {
|
||||
self.method = method;
|
||||
RequestBuilder { url: self.url, method: self.method, headers: self.headers, body: self.body, _state: PhantomData }
|
||||
}
|
||||
}
|
||||
|
||||
// .header(), .body() available in any state that has at least URL.
|
||||
impl<S> RequestBuilder<S> {
|
||||
pub fn header(mut self, name: HeaderName, value: HeaderValue) -> Self {
|
||||
self.headers.insert(name, value);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
// .send() only available once URL and method are set.
|
||||
impl RequestBuilder<Ready> {
|
||||
pub async fn send(self, client: &Client) -> reqwest::Result<Response> { /* ... */ }
|
||||
}
|
||||
```
|
||||
|
||||
`client.send(RequestBuilder::new().send(...))` - compile error. The agent has to fill in the required steps. The IDE autocomplete also reflects only the legal next steps.
|
||||
|
||||
### File Handles
|
||||
|
||||
```rust
|
||||
pub struct File<State> {
|
||||
fd: RawFd,
|
||||
_state: PhantomData<State>,
|
||||
}
|
||||
|
||||
pub struct Open;
|
||||
pub struct Locked;
|
||||
pub struct Closed;
|
||||
|
||||
impl File<Open> {
|
||||
pub fn open(path: &Path) -> std::io::Result<Self> { /* ... */ }
|
||||
pub fn lock_exclusive(self) -> std::io::Result<File<Locked>> { /* flock */ }
|
||||
pub fn close(self) -> std::io::Result<File<Closed>> { /* ... */ }
|
||||
}
|
||||
|
||||
impl File<Locked> {
|
||||
pub fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> { /* ... */ }
|
||||
pub fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { /* ... */ }
|
||||
pub fn unlock(self) -> std::io::Result<File<Open>> { /* ... */ }
|
||||
}
|
||||
|
||||
impl File<Closed> {
|
||||
// No methods. The type only exists to be dropped.
|
||||
}
|
||||
```
|
||||
|
||||
You cannot `read()` an unlocked file. You cannot `close()` while holding a lock without unlocking first. You cannot use a closed file at all - it has no methods.
|
||||
|
||||
## Sealed Traits
|
||||
|
||||
Sometimes you want a closed set of types implementing a trait, defined by the crate, not extensible by downstream. The sealed trait pattern:
|
||||
|
||||
```rust
|
||||
mod sealed {
|
||||
pub trait Sealed {}
|
||||
}
|
||||
|
||||
pub trait Renderer: sealed::Sealed {
|
||||
fn render(&self, frame: &mut Frame);
|
||||
}
|
||||
|
||||
pub struct OpenGl;
|
||||
pub struct Vulkan;
|
||||
pub struct Metal;
|
||||
|
||||
impl sealed::Sealed for OpenGl {}
|
||||
impl sealed::Sealed for Vulkan {}
|
||||
impl sealed::Sealed for Metal {}
|
||||
|
||||
impl Renderer for OpenGl { fn render(&self, f: &mut Frame) { /* ... */ } }
|
||||
impl Renderer for Vulkan { fn render(&self, f: &mut Frame) { /* ... */ } }
|
||||
impl Renderer for Metal { fn render(&self, f: &mut Frame) { /* ... */ } }
|
||||
```
|
||||
|
||||
Downstream code cannot add new `impl Renderer for Whatever` because they cannot implement `sealed::Sealed` (its module is private). Useful when you want trait dispatch but maintain the invariant that you control all implementations.
|
||||
|
||||
## NonEmpty Collections
|
||||
|
||||
```rust
|
||||
pub struct NonEmptyVec<T> {
|
||||
head: T,
|
||||
tail: Vec<T>,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("vector was empty")]
|
||||
pub struct Empty;
|
||||
|
||||
impl<T> NonEmptyVec<T> {
|
||||
pub fn try_from_vec(mut v: Vec<T>) -> Result<Self, Empty> {
|
||||
if v.is_empty() { return Err(Empty); }
|
||||
let tail = v.split_off(1);
|
||||
let head = v.into_iter().next().expect("checked non-empty");
|
||||
Ok(Self { head, tail })
|
||||
}
|
||||
|
||||
pub fn first(&self) -> &T { &self.head }
|
||||
pub fn len(&self) -> usize { self.tail.len() + 1 }
|
||||
}
|
||||
```
|
||||
|
||||
Functions taking `NonEmptyVec<T>` cannot receive an empty vector. The `first()` method returns `&T`, not `Option<&T>`. The agent never has to write `match v.first() { Some(x) => ..., None => panic!(...) }` again.
|
||||
|
||||
## When NOT to Newtype
|
||||
|
||||
- For one-off internal computations where the unit lives in a single function and never crosses a boundary.
|
||||
- When the wrapper does not change behavior or invariants vs. the underlying type (e.g., a `struct Count(u32)` that is only ever used in one struct).
|
||||
- When `From`/`Into` conversions would be ergonomic but would defeat the purpose (if you find yourself wanting `impl From<UserId> for Uuid`, you do not want a newtype - you want a type alias).
|
||||
|
||||
The cost of a newtype is one tuple struct + the impls you need. The break-even is around three uses across different functions, or any use that crosses an API boundary.
|
||||
|
||||
## When NOT to Use Type-State
|
||||
|
||||
- When the state space is small and transitions are simple (`Option<T>` and `Result<T, E>` are state machines already).
|
||||
- When the type-state would force runtime branching upward (e.g., reading "should this run as Open or Locked?" from config means you store `Box<dyn FileLike>` anyway).
|
||||
- When the API is consumed by code that does not know the state at compile time (heterogeneous collections, dynamic dispatch boundaries).
|
||||
|
||||
In those cases, regular enum-tagged states are correct. The line is: **can the call site know the state statically?** If yes, type-state. If no, enum-with-tag.
|
||||
@@ -0,0 +1,250 @@
|
||||
# Unsafe Discipline
|
||||
|
||||
The reason Chris Allen's "implementing a persistent memory arena in Rust was not hard" works: the unsafe surface area is microscopic, it lives behind one newtype with one constructor, and every block has a SAFETY comment that names a specific invariant. Coding agents follow the pattern mechanically once the shape is established.
|
||||
|
||||
## The Three Required Components
|
||||
|
||||
Every `unsafe` block needs all three. No exceptions.
|
||||
|
||||
1. **Safe wrapper.** No `unsafe fn` or raw pointer types in the crate's public API. If a caller needs to construct an instance, the constructor either does the work safely or is `unsafe` with a documented contract.
|
||||
2. **SAFETY comment.** A `// SAFETY:` line within 5 lines above the `unsafe { ... }` block, stating which invariant is upheld and where it comes from. Generic phrases ("this is safe because we checked") fail review.
|
||||
3. **miri proof.** A test that exercises the unsafe path under `cargo +nightly miri nextest run`. If the path cannot be exercised under miri (FFI, syscalls), provide an alternate proof and gate behind a feature flag ending in `-skip-miri`.
|
||||
|
||||
## The Wrapper Pattern (`NonNull<T>` style)
|
||||
|
||||
Reference the screenshot Chris Allen quoted - `std::ptr::NonNull<T>`. Mirror this shape for every raw pointer, raw slice, raw transmute, or uninit memory operation in your own code.
|
||||
|
||||
```rust
|
||||
use core::marker::PhantomData;
|
||||
use core::ptr::NonNull;
|
||||
|
||||
/// A non-null, properly aligned, initialized pointer that does not alias.
|
||||
///
|
||||
/// All invariants are upheld by [`Self::new`] (checked) or [`Self::new_unchecked`]
|
||||
/// (delegated to the caller's contract). Once you hold an `InitPtr<T>`, every
|
||||
/// public method on it is safe to call.
|
||||
#[derive(Debug)]
|
||||
#[repr(transparent)]
|
||||
pub struct InitPtr<T> {
|
||||
inner: NonNull<T>,
|
||||
_marker: PhantomData<T>,
|
||||
}
|
||||
|
||||
// Send/Sync are NOT automatic for raw-pointer-bearing types. Decide deliberately.
|
||||
// SAFETY: `InitPtr<T>` owns no concurrency state of its own; whether it is
|
||||
// Send/Sync depends on `T`. The bounds below mirror `Box<T>`.
|
||||
unsafe impl<T: Send> Send for InitPtr<T> {}
|
||||
unsafe impl<T: Sync> Sync for InitPtr<T> {}
|
||||
|
||||
impl<T> InitPtr<T> {
|
||||
/// Wrap a raw pointer after checking alignment and non-null. The
|
||||
/// initialization invariant is not statically checkable here; callers must
|
||||
/// only feed pointers to memory that was written before this call.
|
||||
pub fn new(ptr: *mut T) -> Option<Self> {
|
||||
if !ptr.is_aligned() {
|
||||
return None;
|
||||
}
|
||||
// SAFETY: alignment checked above; `NonNull::new` filters null. The
|
||||
// caller is documented to provide an initialized location.
|
||||
NonNull::new(ptr).map(|inner| Self { inner, _marker: PhantomData })
|
||||
}
|
||||
|
||||
/// Wrap a raw pointer the caller asserts is valid.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// - `ptr` is non-null.
|
||||
/// - `ptr` is aligned to `align_of::<T>()`.
|
||||
/// - `*ptr` is initialized at the time of this call.
|
||||
/// - For the lifetime of the returned value, no other `&T` or `&mut T`
|
||||
/// aliases `*ptr`.
|
||||
pub unsafe fn new_unchecked(ptr: *mut T) -> Self {
|
||||
// SAFETY: caller upholds non-null per the function contract.
|
||||
Self { inner: unsafe { NonNull::new_unchecked(ptr) }, _marker: PhantomData }
|
||||
}
|
||||
|
||||
pub fn as_ref(&self) -> &T {
|
||||
// SAFETY: the constructor's invariants guarantee `inner` points at an
|
||||
// initialized, aligned, non-aliased `T`. Reborrowing through `&self`
|
||||
// ties the resulting lifetime to `self`, enforcing the rest via
|
||||
// standard borrow rules.
|
||||
unsafe { self.inner.as_ref() }
|
||||
}
|
||||
|
||||
pub fn as_mut(&mut self) -> &mut T {
|
||||
// SAFETY: `&mut self` proves no other reference can alias `inner` for
|
||||
// the lifetime of the returned `&mut T`; remaining invariants come
|
||||
// from construction.
|
||||
unsafe { self.inner.as_mut() }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The list of features this single shape gives you:
|
||||
|
||||
- The agent cannot construct `InitPtr<T>` without going through a checked path or accepting the `unsafe` obligation explicitly.
|
||||
- The agent cannot leak the raw pointer; `as_ref` / `as_mut` return safe references with proper lifetimes.
|
||||
- The agent cannot accidentally Send/Sync where it shouldn't - the `unsafe impl` is explicit per-bound.
|
||||
- `#[repr(transparent)]` means the type is layout-compatible with `*mut T` for FFI, without exposing the raw pointer.
|
||||
|
||||
## SAFETY Comment Grammar
|
||||
|
||||
Every comment maps one-to-one to an invariant. Format:
|
||||
|
||||
```rust
|
||||
// SAFETY: <which invariant is upheld>: <how it is established here>.
|
||||
```
|
||||
|
||||
Anti-examples (do not pass review):
|
||||
|
||||
```rust
|
||||
// SAFETY: this is fine
|
||||
// SAFETY: we know what we're doing
|
||||
// SAFETY: the caller will not pass null
|
||||
// SAFETY: tested
|
||||
```
|
||||
|
||||
Good examples:
|
||||
|
||||
```rust
|
||||
// SAFETY: `len <= self.capacity` was checked at line 87 and `self.ptr` was
|
||||
// allocated by the same allocator we are reading through.
|
||||
|
||||
// SAFETY: `read_volatile` requires alignment and non-null; both hold because
|
||||
// `self.inner` is an `InitPtr<T>` whose constructor enforced them.
|
||||
|
||||
// SAFETY: We hold `&mut self`, so no concurrent reader exists. The slice
|
||||
// reference is dropped before the next `&self` borrow because we shrink the
|
||||
// returned scope manually.
|
||||
```
|
||||
|
||||
## Persistent Memory Arena Pattern (the Chris Allen example)
|
||||
|
||||
A persistent memory arena (PMA) is an `mmap`-backed bump allocator that survives process restarts. It is the classic "lots of unsafe under one safe surface" project.
|
||||
|
||||
Shape:
|
||||
|
||||
```rust
|
||||
pub struct Arena {
|
||||
map: Mmap, // wraps `mmap(2)` - safe wrapper from `memmap2` crate
|
||||
head: AtomicUsize, // current bump offset, atomic for multi-writer if needed
|
||||
}
|
||||
|
||||
impl Arena {
|
||||
pub fn open(path: &Path, capacity: usize) -> std::io::Result<Self> { /* mmap, init header */ }
|
||||
|
||||
pub fn alloc<T>(&self, value: T) -> Result<Handle<T>, ArenaFull> {
|
||||
let layout = Layout::new::<T>();
|
||||
let aligned = align_up(self.head.load(Acquire), layout.align());
|
||||
let next = aligned.checked_add(layout.size()).ok_or(ArenaFull)?;
|
||||
if next > self.map.len() { return Err(ArenaFull); }
|
||||
// CAS the head forward; retry on contention.
|
||||
// ... omitted for brevity ...
|
||||
|
||||
// SAFETY: `aligned + layout.size() <= self.map.len()` was just proven.
|
||||
// The mmap region is exclusively owned by this arena while we hold the
|
||||
// bump. `aligned` is aligned to `layout.align()` by `align_up`. No
|
||||
// other writer can have observed this offset because the CAS above
|
||||
// returned `Ok`.
|
||||
let ptr = unsafe { self.map.as_mut_ptr().add(aligned) as *mut T };
|
||||
// SAFETY: `ptr` is non-null (mmap base + offset), aligned (above),
|
||||
// exclusively owned (CAS), and we are about to initialize it.
|
||||
unsafe { ptr::write(ptr, value) };
|
||||
|
||||
// SAFETY: same invariants; we wrap the now-initialized pointer in the
|
||||
// safe handle which encapsulates further accesses.
|
||||
Ok(unsafe { Handle::new_unchecked(ptr, self) })
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Handle<'a, T> {
|
||||
inner: InitPtr<T>,
|
||||
_arena: PhantomData<&'a Arena>,
|
||||
}
|
||||
```
|
||||
|
||||
Three `unsafe` blocks, three SAFETY comments, one safe handle type emerging on the other side. The agent now uses `Handle<T>` everywhere - never `*mut T`.
|
||||
|
||||
## Miri Invocation
|
||||
|
||||
```bash
|
||||
# install once
|
||||
rustup install nightly
|
||||
rustup component add miri rust-src --toolchain nightly
|
||||
|
||||
# run on every change that touches unsafe
|
||||
MIRIFLAGS="-Zmiri-strict-provenance -Zmiri-symbolic-alignment-check" \
|
||||
cargo +nightly miri nextest run --all-features
|
||||
```
|
||||
|
||||
What miri catches that the borrow checker cannot:
|
||||
|
||||
- Use-after-free
|
||||
- Double-free
|
||||
- Reads of uninitialized memory
|
||||
- Pointer-from-integer reconstruction that violates strict provenance
|
||||
- Alignment lies (transmuting unaligned data)
|
||||
- Stacked borrows / Tree borrows aliasing violations
|
||||
- Data races (single-threaded model, but catches concurrent access through `UnsafeCell` misuse)
|
||||
- Atomic ordering bugs in some patterns
|
||||
- Memory leaks (with `-Zmiri-track-pointer-tag`)
|
||||
|
||||
## When Miri Cannot Run
|
||||
|
||||
Certain paths are off-limits for miri: most syscalls beyond a curated allowlist, real network I/O, `std::process` calls, OS-specific FFI, hardware-dependent intrinsics on non-x86. Strategy:
|
||||
|
||||
1. **Isolate.** Put the un-mirifiable code in its own module behind `#[cfg(feature = "ffi-real")]` or similar.
|
||||
2. **Mock at the boundary.** For everything below the FFI boundary, write a safe Rust fake (a `Vec<u8>`-backed "disk", a fake clock, an in-memory socket pair). Expose it as a trait the production code consumes.
|
||||
3. **Test the fake under miri.** The fake implementation exercises the same logic minus the syscall. If the logic is unsafe (raw pointer manipulation in the fake "disk" buffer), miri catches the bug.
|
||||
4. **Test the real path under regular `cargo test`.** With `cargo nextest run --features ffi-real`. No miri, but the surface area is now just the syscall boundary.
|
||||
5. **Document.** A `# Safety` section in the rustdoc names the obligations the FFI puts on us, and a `# Testing` section explains the mock-vs-real split.
|
||||
|
||||
## Loom for Concurrency
|
||||
|
||||
When `unsafe` participates in a concurrent algorithm (lock-free queue, hazard pointers, custom Arc), miri's single-thread model is insufficient. Use `loom`:
|
||||
|
||||
```rust
|
||||
#[cfg(loom)]
|
||||
use loom::sync::atomic::{AtomicUsize, Ordering};
|
||||
#[cfg(not(loom))]
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
#[cfg(loom)]
|
||||
fn concurrent_push_pop() {
|
||||
loom::model(|| {
|
||||
let queue = std::sync::Arc::new(MyQueue::new());
|
||||
let q1 = queue.clone();
|
||||
let q2 = queue.clone();
|
||||
let h1 = loom::thread::spawn(move || q1.push(1));
|
||||
let h2 = loom::thread::spawn(move || q2.pop());
|
||||
h1.join().unwrap();
|
||||
h2.join().unwrap();
|
||||
});
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Run: `RUSTFLAGS="--cfg loom" cargo test --release`. Loom exhaustively explores thread interleavings for the test scope. Combined with miri on the single-thread paths, you have machine-checked soundness over the full state space.
|
||||
|
||||
## The Forbidden List
|
||||
|
||||
Reject in code review, automatic CI fail:
|
||||
|
||||
- `unsafe { ... }` with no SAFETY comment within 5 lines above.
|
||||
- `unsafe { unsafe_op_a(); unsafe_op_b(); }` (multiple unsafe ops in one block - split them, one SAFETY each). Clippy: `multiple_unsafe_ops_per_block`.
|
||||
- `unsafe fn` exposed publicly without a documented `# Safety` section in rustdoc.
|
||||
- `std::mem::transmute` for anything but lifetime extension on the same layout (and that should usually be `core::mem::transmute_copy` or `bytemuck::cast` if the relayout is well-defined).
|
||||
- `std::ptr::read_unaligned` / `write_unaligned` without a comment explaining why aligned access is impossible.
|
||||
- `from_raw_parts` / `from_raw_parts_mut` without proving the source pointer's provenance covers the entire slice.
|
||||
- `Arc::get_mut_unchecked`, `Box::leak` to bypass ownership, `MaybeUninit::assume_init` on partially-initialized data.
|
||||
- `unsafe impl Send`, `unsafe impl Sync` on types containing raw pointers, without a comment naming exactly which interior-mutability rule is upheld.
|
||||
- Any `unsafe` block whose justification depends on "in practice this never happens".
|
||||
|
||||
## The One-Line Summary
|
||||
|
||||
> Wrap once. Prove once. Test under miri. Never let `unsafe` escape.
|
||||
@@ -0,0 +1,527 @@
|
||||
# Zero-Cost Safety — Zig Ergonomics in Rust
|
||||
|
||||
Rust already owns memory safety. This reference adds the patterns that give you Zig's *ergonomic* safety — explicit allocation control, compile-time computation, zero-hidden-cost APIs, bit-level layout, and deterministic cleanup — without leaving the Rust toolchain.
|
||||
|
||||
**When to load this file:** arena, allocator, bumpalo, const fn, const generics, comptime, zero-alloc, no-alloc, slice-based API, `#[repr]`, packed struct, bitfield, scopeguard, errdefer, RAII cleanup, Zig-like patterns.
|
||||
|
||||
---
|
||||
|
||||
## 1. Explicit Allocators — Arena Pattern
|
||||
|
||||
Zig passes `allocator: Allocator` to every function. Rust's stable equivalent: arena crates that make allocation scope visible and bulk-freeable.
|
||||
|
||||
### bumpalo — The Default Arena
|
||||
|
||||
```rust
|
||||
use bumpalo::Bump;
|
||||
|
||||
fn parse_tokens<'a>(arena: &'a Bump, input: &[u8]) -> Vec<&'a str> {
|
||||
// All allocations go into `arena`. Caller controls lifetime.
|
||||
// When `arena` drops, everything frees in one shot.
|
||||
let token = arena.alloc_str("hello");
|
||||
let slice = arena.alloc_slice_copy(&[1u8, 2, 3]);
|
||||
vec![token] // Vec itself is on heap; contents point into arena
|
||||
}
|
||||
|
||||
// Usage: caller owns the arena, decides when memory dies.
|
||||
let arena = Bump::new();
|
||||
let tokens = parse_tokens(&arena, b"...");
|
||||
drop(arena); // all arena memory freed, zero individual deallocations
|
||||
```
|
||||
|
||||
**When to use:** parsers, compilers, game frame allocators, request-scoped web handlers, any hot loop where individual `Box`/`Vec` alloc+free overhead matters.
|
||||
|
||||
### typed-arena — Homogeneous Arena
|
||||
|
||||
```rust
|
||||
use typed_arena::Arena;
|
||||
|
||||
struct AstNode { kind: u8, children: Vec<&'static AstNode> } // simplified
|
||||
|
||||
let node_arena: Arena<AstNode> = Arena::new();
|
||||
let root = node_arena.alloc(AstNode { kind: 0, children: vec![] });
|
||||
// All nodes share arena lifetime. No individual free.
|
||||
```
|
||||
|
||||
**When to use:** tree/graph structures where all nodes have the same type and same lifetime.
|
||||
|
||||
### allocator_api (nightly) — Full Zig Parity
|
||||
|
||||
```rust
|
||||
#![feature(allocator_api)]
|
||||
use std::alloc::Global;
|
||||
|
||||
// Vec parameterized by allocator — exactly like Zig.
|
||||
let v: Vec<u8, &Bump> = Vec::new_in(&arena);
|
||||
|
||||
// Custom allocator for tracking, limiting, or redirecting allocation
|
||||
struct CountingAlloc { inner: Global, count: AtomicUsize }
|
||||
unsafe impl Allocator for CountingAlloc { /* ... */ }
|
||||
```
|
||||
|
||||
**When to use:** when you need allocator-generic data structures on nightly. For stable code, prefer `bumpalo` directly.
|
||||
|
||||
### Decision Tree
|
||||
|
||||
```
|
||||
Need arena allocation?
|
||||
├── All items same type, same lifetime → typed-arena
|
||||
├── Mixed types, same lifetime → bumpalo
|
||||
├── Need allocator-generic containers → allocator_api (nightly)
|
||||
└── Just need fewer allocations → SmallVec / ArrayVec / tinyvec (stack-first)
|
||||
```
|
||||
|
||||
### Cargo.toml
|
||||
|
||||
```toml
|
||||
bumpalo = { version = "3", features = ["collections"] }
|
||||
typed-arena = "2"
|
||||
smallvec = { version = "1", features = ["union", "const_generics"] }
|
||||
tinyvec = { version = "1", features = ["alloc"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. Compile-Time Computation — const fn, const generics, proc macros
|
||||
|
||||
Zig's `comptime` runs arbitrary code at compile time. Rust splits this across three mechanisms.
|
||||
|
||||
### const fn — Compile-Time Pure Functions
|
||||
|
||||
```rust
|
||||
const fn fibonacci(n: usize) -> usize {
|
||||
match n {
|
||||
0 => 0,
|
||||
1 => 1,
|
||||
_ => fibonacci(n - 1) + fibonacci(n - 2),
|
||||
}
|
||||
}
|
||||
|
||||
const FIB_20: usize = fibonacci(20); // computed at compile time: 6765
|
||||
|
||||
// Use in array sizes
|
||||
const LOOKUP: [u8; 256] = {
|
||||
let mut table = [0u8; 256];
|
||||
let mut i = 0;
|
||||
while i < 256 {
|
||||
table[i] = (i as u8).wrapping_mul(7);
|
||||
i += 1;
|
||||
}
|
||||
table
|
||||
};
|
||||
```
|
||||
|
||||
**Stable since Rust 1.82:** `const fn` supports `match`, loops, `if`, references, mutable locals — nearly full Rust. Use `const { }` blocks (Rust 1.79+) for inline compile-time assertions.
|
||||
|
||||
```rust
|
||||
fn process<const N: usize>(data: &[u8; N]) {
|
||||
const { assert!(N > 0, "N must be positive") }; // compile-time panic if N == 0
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
### const generics — Type-Level Values
|
||||
|
||||
```rust
|
||||
struct Buffer<const N: usize> {
|
||||
data: [u8; N],
|
||||
len: usize,
|
||||
}
|
||||
|
||||
impl<const N: usize> Buffer<N> {
|
||||
const fn new() -> Self {
|
||||
Self { data: [0; N], len: 0 }
|
||||
}
|
||||
|
||||
fn push(&mut self, byte: u8) -> Result<(), BufferFullError> {
|
||||
if self.len >= N { return Err(BufferFullError); }
|
||||
self.data[self.len] = byte;
|
||||
self.len += 1;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// Compiler enforces: Buffer<16> and Buffer<32> are distinct types.
|
||||
let small: Buffer<16> = Buffer::new();
|
||||
let large: Buffer<1024> = Buffer::new();
|
||||
```
|
||||
|
||||
### proc macros — Code Generation (Zig comptime type creation)
|
||||
|
||||
When `const fn` is not enough (generating struct fields, impl blocks, or derive logic), proc macros fill the gap.
|
||||
|
||||
```rust
|
||||
// In a proc-macro crate:
|
||||
use proc_macro::TokenStream;
|
||||
use quote::quote;
|
||||
use syn::{parse_macro_input, DeriveInput};
|
||||
|
||||
#[proc_macro_derive(Builder)]
|
||||
pub fn derive_builder(input: TokenStream) -> TokenStream {
|
||||
let input = parse_macro_input!(input as DeriveInput);
|
||||
let name = &input.ident;
|
||||
// ... generate builder struct and impl
|
||||
TokenStream::from(quote! {
|
||||
impl #name {
|
||||
pub fn builder() -> #name##Builder { /* ... */ }
|
||||
}
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
**Decision tree:**
|
||||
|
||||
```
|
||||
Need compile-time value computation? → const fn
|
||||
Need type parameterized by value? → const generics
|
||||
Need to generate new types/impls? → proc macro (derive or attribute)
|
||||
Need compile-time string processing? → proc macro
|
||||
Need typenum-level arithmetic? → typenum / generic-array (rare)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Zero-Allocation API Design — No Hidden Costs
|
||||
|
||||
Zig's philosophy: no operator overloading, no hidden allocation, every cost visible. Rust achieves this with discipline.
|
||||
|
||||
### Slice-Based APIs — Caller Owns Memory
|
||||
|
||||
```rust
|
||||
// BAD: hidden allocation in return type
|
||||
fn process(input: &str) -> String {
|
||||
input.to_uppercase() // allocates
|
||||
}
|
||||
|
||||
// GOOD: caller provides output buffer, zero allocation
|
||||
fn process(input: &[u8], output: &mut [u8]) -> usize {
|
||||
let len = input.len().min(output.len());
|
||||
for i in 0..len {
|
||||
output[i] = input[i].to_ascii_uppercase();
|
||||
}
|
||||
len // returns bytes written
|
||||
}
|
||||
|
||||
// GOOD: return borrowed data when possible
|
||||
fn find_token<'a>(input: &'a str) -> Option<&'a str> {
|
||||
input.split_whitespace().next() // no allocation — borrows from input
|
||||
}
|
||||
```
|
||||
|
||||
### try_* APIs — Fallible Allocation
|
||||
|
||||
```rust
|
||||
// Allocation can fail explicitly (like Zig's allocator returning error)
|
||||
let mut v = Vec::new();
|
||||
v.try_reserve(1_000_000)?; // returns Result, not panic
|
||||
|
||||
// For Box:
|
||||
let b = Box::try_new(42)?; // nightly, or use allocator_api
|
||||
```
|
||||
|
||||
### SmallVec / ArrayVec — Stack-First Collections
|
||||
|
||||
```rust
|
||||
use smallvec::SmallVec;
|
||||
use arrayvec::ArrayVec;
|
||||
|
||||
// SmallVec: stack for small counts, heap spillover for large
|
||||
let mut tags: SmallVec<[u8; 8]> = SmallVec::new();
|
||||
tags.push(1); // on stack if <= 8 elements
|
||||
|
||||
// ArrayVec: purely stack, fixed capacity, no heap ever
|
||||
let mut buf: ArrayVec<u8, 64> = ArrayVec::new();
|
||||
buf.try_push(42).map_err(|_| "full")?; // returns error instead of panic
|
||||
```
|
||||
|
||||
### Cow — Defer Allocation Until Mutation
|
||||
|
||||
```rust
|
||||
use std::borrow::Cow;
|
||||
|
||||
fn normalize(input: &str) -> Cow<'_, str> {
|
||||
if input.contains('\t') {
|
||||
Cow::Owned(input.replace('\t', " ")) // allocates only when needed
|
||||
} else {
|
||||
Cow::Borrowed(input) // zero-cost pass-through
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### The #![no_std] Discipline
|
||||
|
||||
For maximum allocation control, go `#![no_std]`:
|
||||
|
||||
```rust
|
||||
#![no_std]
|
||||
extern crate alloc; // opt-in to heap when needed
|
||||
|
||||
use alloc::vec::Vec; // explicit: I chose to allocate
|
||||
use alloc::string::String; // explicit: I chose to allocate
|
||||
```
|
||||
|
||||
Even in `std` code, the *mindset* applies: prefer `&[T]` over `Vec<T>` in function signatures, `&str` over `String`, `&Path` over `PathBuf`.
|
||||
|
||||
### Clippy Lints for Hidden Allocations
|
||||
|
||||
```toml
|
||||
# Cargo.toml — catch accidental allocations
|
||||
[lints.clippy]
|
||||
# These warn on patterns that allocate when a borrow would suffice:
|
||||
unnecessary_to_owned = "warn" # .to_string() / .to_vec() when borrow works
|
||||
redundant_clone = "warn" # .clone() that's immediately consumed
|
||||
large_stack_arrays = "warn" # accidental large stack usage
|
||||
vec_init_then_push = "warn" # Vec::new() + push instead of vec![]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Bit-Level Layout — repr, Packed Structs, Bitfields
|
||||
|
||||
Zig: `packed struct` with bit-level field control. Rust matches with `#[repr]` attributes and bitfield crates.
|
||||
|
||||
### #[repr(C)] — Guaranteed C-Compatible Layout
|
||||
|
||||
```rust
|
||||
#[repr(C)]
|
||||
struct Header {
|
||||
magic: [u8; 4],
|
||||
version: u16,
|
||||
flags: u16,
|
||||
length: u32,
|
||||
}
|
||||
// Layout is C ABI: fields in declaration order, C padding rules.
|
||||
// Safe to transmute from/to byte arrays via zerocopy.
|
||||
```
|
||||
|
||||
### #[repr(C, packed)] — No Padding
|
||||
|
||||
```rust
|
||||
#[repr(C, packed)]
|
||||
struct WireHeader {
|
||||
tag: u8,
|
||||
length: u16, // NOT aligned to 2-byte boundary
|
||||
checksum: u32,
|
||||
}
|
||||
// Total size: exactly 7 bytes. No padding.
|
||||
// WARNING: taking &self.length is UB if unaligned. Use read_unaligned or zerocopy.
|
||||
```
|
||||
|
||||
**Safe access pattern:**
|
||||
|
||||
```rust
|
||||
use std::ptr;
|
||||
|
||||
impl WireHeader {
|
||||
fn length(&self) -> u16 {
|
||||
// SAFETY: packed field may be unaligned; ptr::read_unaligned handles this.
|
||||
unsafe { ptr::read_unaligned(ptr::addr_of!(self.length)) }
|
||||
}
|
||||
}
|
||||
|
||||
// Better: use zerocopy to avoid manual unsafe entirely
|
||||
use zerocopy::{FromBytes, IntoBytes, KnownLayout, Immutable};
|
||||
|
||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable)]
|
||||
#[repr(C, packed)]
|
||||
struct WireHeader {
|
||||
tag: u8,
|
||||
length: [u8; 2], // manual byte array avoids alignment issues
|
||||
checksum: [u8; 4],
|
||||
}
|
||||
|
||||
impl WireHeader {
|
||||
fn length(&self) -> u16 { u16::from_le_bytes(self.length) }
|
||||
fn checksum(&self) -> u32 { u32::from_le_bytes(self.checksum) }
|
||||
}
|
||||
```
|
||||
|
||||
### bitfield — Bit-Level Flag Packing
|
||||
|
||||
```rust
|
||||
use bitfield::bitfield;
|
||||
|
||||
bitfield! {
|
||||
pub struct Permissions(u8);
|
||||
impl Debug;
|
||||
pub bool, readable, set_readable: 0;
|
||||
pub bool, writable, set_writable: 1;
|
||||
pub bool, executable, set_executable: 2;
|
||||
pub u8, level, set_level: 5, 3; // bits 3-5
|
||||
}
|
||||
|
||||
let mut p = Permissions(0);
|
||||
p.set_readable(true);
|
||||
p.set_level(5);
|
||||
assert!(p.readable());
|
||||
assert_eq!(p.level(), 5);
|
||||
```
|
||||
|
||||
### modular-bitfield — Richer Bitfield API
|
||||
|
||||
```rust
|
||||
use modular_bitfield::prelude::*;
|
||||
|
||||
#[bitfield(bits = 16)]
|
||||
#[derive(Debug)]
|
||||
pub struct StatusWord {
|
||||
ready: bool, // 1 bit
|
||||
error_code: B4, // 4 bits
|
||||
#[skip] __: B3, // 3 bits padding
|
||||
priority: B8, // 8 bits
|
||||
}
|
||||
```
|
||||
|
||||
### zerocopy — Safe Zero-Copy Parsing
|
||||
|
||||
```rust
|
||||
use zerocopy::{FromBytes, IntoBytes, KnownLayout, Immutable, Ref};
|
||||
|
||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable)]
|
||||
#[repr(C)]
|
||||
struct Packet {
|
||||
header: [u8; 4],
|
||||
payload_len: u32,
|
||||
}
|
||||
|
||||
fn parse(bytes: &[u8]) -> Option<&Packet> {
|
||||
Ref::<_, Packet>::from_prefix(bytes).map(|(pkt, _rest)| pkt.into_ref()).ok()
|
||||
}
|
||||
// Zero-copy, zero-allocation, fully safe. No transmute, no pointer cast.
|
||||
```
|
||||
|
||||
### Cargo.toml
|
||||
|
||||
```toml
|
||||
zerocopy = { version = "0.8", features = ["derive"] }
|
||||
bitfield = "0.17"
|
||||
modular-bitfield = "0.11"
|
||||
bytemuck = { version = "1", features = ["derive"] } # alternative to zerocopy
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Scope Guards — errdefer / Deterministic Cleanup
|
||||
|
||||
Zig's `errdefer` runs cleanup only on error paths. Rust's `Drop` always runs, but `scopeguard` gives fine-grained control.
|
||||
|
||||
### scopeguard — The errdefer Equivalent
|
||||
|
||||
```rust
|
||||
use scopeguard::{defer, guard};
|
||||
use std::fs;
|
||||
|
||||
fn create_and_process(path: &str) -> std::io::Result<()> {
|
||||
let file = fs::File::create(path)?;
|
||||
// If anything below fails, clean up the file.
|
||||
// This is exactly Zig's errdefer.
|
||||
let _cleanup = guard((), |_| {
|
||||
let _ = fs::remove_file(path);
|
||||
});
|
||||
|
||||
write_data(&file)?;
|
||||
validate_data(path)?;
|
||||
|
||||
// Success: defuse the guard so it does NOT run cleanup.
|
||||
std::mem::forget(_cleanup);
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
### defer! — Always-Run Cleanup (like Zig's defer)
|
||||
|
||||
```rust
|
||||
use scopeguard::defer;
|
||||
|
||||
fn with_temp_dir() -> anyhow::Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
defer! {
|
||||
// Runs when scope exits, success or failure.
|
||||
println!("Cleaning up {}", dir.path().display());
|
||||
// dir's Drop also cleans up, but this shows the pattern.
|
||||
}
|
||||
|
||||
do_work(dir.path())?;
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
### Drop as RAII Cleanup
|
||||
|
||||
```rust
|
||||
struct TempFile { path: std::path::PathBuf }
|
||||
|
||||
impl TempFile {
|
||||
fn new(path: impl Into<std::path::PathBuf>) -> std::io::Result<Self> {
|
||||
let path = path.into();
|
||||
std::fs::File::create(&path)?;
|
||||
Ok(Self { path })
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TempFile {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_file(&self.path);
|
||||
}
|
||||
}
|
||||
|
||||
// Usage: file is auto-cleaned when `tmp` goes out of scope.
|
||||
let tmp = TempFile::new("/tmp/scratch.dat")?;
|
||||
```
|
||||
|
||||
### The errdefer Pattern — Defuse on Success
|
||||
|
||||
The key insight from Zig's `errdefer`: you want cleanup on error but NOT on success. In Rust:
|
||||
|
||||
```rust
|
||||
use scopeguard::ScopeGuard;
|
||||
|
||||
fn deploy(artifact: &Path) -> Result<(), DeployError> {
|
||||
let backup = backup_current()?;
|
||||
|
||||
// errdefer: restore backup if anything fails
|
||||
let rollback = guard(backup.clone(), |b| {
|
||||
let _ = restore_from_backup(&b);
|
||||
});
|
||||
|
||||
upload(artifact)?;
|
||||
health_check()?;
|
||||
|
||||
// Success path: defuse the rollback guard
|
||||
ScopeGuard::into_inner(rollback);
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
### Cargo.toml
|
||||
|
||||
```toml
|
||||
scopeguard = "1"
|
||||
tempfile = "3" # idiomatic RAII temp files/dirs
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Summary: Zig Advantage → Rust Pattern
|
||||
|
||||
| Zig Feature | Rust Equivalent | Difficulty | Reference |
|
||||
|---|---|---|---|
|
||||
| Explicit allocator passing | `bumpalo` / `typed-arena` / `allocator_api` | Easy | §1 |
|
||||
| `comptime` value computation | `const fn` + `const { }` blocks | Easy | §2 |
|
||||
| `comptime` type generation | proc macros (derive / attribute) | Medium | §2 |
|
||||
| No hidden allocations | `#![no_std]` / slice-based APIs / `Cow` | Style choice | §3 |
|
||||
| `packed struct` / bitfields | `#[repr(C, packed)]` / `bitfield` / `zerocopy` | Easy | §4 |
|
||||
| `errdefer` | `scopeguard::guard` + defuse on success | Easy | §5 |
|
||||
| `defer` | `scopeguard::defer!` / `Drop` | Easy | §5 |
|
||||
|
||||
All achievable within Rust's single toolchain. You get Zig's explicitness **plus** the borrow checker, lifetime analysis, trait bounds, and `miri`. The combination is strictly more powerful than either alone.
|
||||
|
||||
## When NOT to Use These Patterns
|
||||
|
||||
- **Arena allocation** overkill for simple CLI tools that allocate once and exit.
|
||||
- **Zero-alloc APIs** hurt readability when the function naturally produces owned data. Don't force `&mut [u8]` output buffers on a function that logically returns `String`.
|
||||
- **`#[repr(packed)]`** only for wire formats and FFI. Never for regular domain types.
|
||||
- **Scope guards** unnecessary when `Drop` on the value itself handles cleanup (e.g., `tempfile::NamedTempFile` already does this).
|
||||
- **`const fn`** everything? No — only when the value is genuinely needed at compile time or the function is trivially const-eligible. Don't contort logic just to be const.
|
||||
|
||||
The goal is **visible costs and explicit control**, not asceticism. Use `String` and `Vec` freely when they're the right tool. Reach for these patterns when allocation behavior matters for correctness or performance.
|
||||
@@ -0,0 +1,195 @@
|
||||
|
||||
# TypeScript Programmer
|
||||
|
||||
Modern TypeScript. Type-strict, stack-first, async-correct.
|
||||
|
||||
## Philosophy
|
||||
|
||||
The compiler is your proof system. Make illegal states unrepresentable. Parse at boundaries. Every function has a contract; the type system enforces it.
|
||||
|
||||
## Hard rules
|
||||
|
||||
These are deliberate project choices. Violations are always wrong, not "style preferences".
|
||||
|
||||
### Tooling
|
||||
|
||||
| Category | Use | Never |
|
||||
|---|---|---|
|
||||
| Runtime | Bun (native TS, single binary) | ts-node, tsx |
|
||||
| Package manager | `pnpm` | npm, yarn (unless workspace requires it) |
|
||||
| Linter + formatter | Biome | ESLint, Prettier |
|
||||
| Type checker | `tsc --noEmit` with strict config | skip type checking |
|
||||
| Web framework | Hono | Express |
|
||||
| Validation | Zod | joi, yup, class-validator |
|
||||
| Testing | `bun test` or vitest | jest |
|
||||
| ORM | Drizzle | TypeORM, Prisma (unless already in project) |
|
||||
|
||||
### The iron list
|
||||
|
||||
1. **Readonly by default** — all `type`/`interface` properties are `readonly`. Arrays are `readonly T[]`. Mutable only when mutation is the documented purpose.
|
||||
2. **Branded types for distinct IDs** — `type UserId = Brand<string, "UserId">`. Never pass raw `string` where a branded type exists.
|
||||
3. **Exhaustive switch** — every `switch` on a discriminated union ends with `default: assertNever(x)`. No fall-through.
|
||||
4. **No any** — `any` is banned in annotations, returns, and parameters. Use `unknown` and narrow.
|
||||
5. **No type assertions** — `as any`, `as unknown` banned. `as const` and `satisfies` are fine.
|
||||
6. **No non-null assertion** — `x!` is banned. Use narrowing or optional chaining (`x?.y`).
|
||||
7. **No @ts-ignore / @ts-expect-error** — fix the type.
|
||||
8. **No enum** — use `as const` objects + literal union types.
|
||||
9. **Zod at boundaries** — external input (API, user, file) → Zod schema. Internal → plain types.
|
||||
10. **Typed errors** — Error subclasses with typed fields. No `throw new Error("bare string")` for domain errors. Use Result for expected failures within 1-2 call levels; throw for propagation across many layers.
|
||||
11. **as const for constants** — module-level constant objects and arrays use `as const`.
|
||||
12. **import type** — type-only imports use `import type`. Enforced by `verbatimModuleSyntax`.
|
||||
13. **Named exports only** — no `export default`. Exception: framework requirement (Next.js pages, etc.).
|
||||
14. **No empty catch, no catch-and-swallow** — every `catch` block must either (a) narrow the error with `instanceof` and handle each case, or (b) re-throw. Empty catch blocks and `catch (e) { console.error(e) }` without narrowing or re-throw are banned — they hide bugs. At top-level boundaries (CLI entry, HTTP handler), opt out with `// no-excuse-ok: catch`.
|
||||
|
||||
### Data modeling — which construct, when
|
||||
|
||||
| Situation | Use |
|
||||
|---|---|
|
||||
| User input, API request/response | Zod schema + `z.infer` |
|
||||
| Internal value object | `type` with `readonly` properties |
|
||||
| Function with multiple outcomes | Discriminated union (`kind` field) |
|
||||
| Contract for implementations | `interface` |
|
||||
| Fixed constants | `as const` + literal union |
|
||||
| Distinct primitive (UserId vs OrderId) | Branded type |
|
||||
| Key-value map | `Record<K, V>` or index signature |
|
||||
|
||||
**The one rule**: data crosses trust boundary → Zod. Everything else → plain `type` with `readonly`.
|
||||
|
||||
Load `data-modeling.md` for the full decision flowchart and comparison.
|
||||
|
||||
### When readonly does not apply
|
||||
|
||||
- **Framework state** (React `useState`, signals) — managed by framework.
|
||||
- **Builder / accumulator** — object exists to be mutated (buffer, cache). Document why.
|
||||
- **ORM mutations** — Drizzle insert/update objects.
|
||||
|
||||
### Why empty/unhandled catch is banned
|
||||
|
||||
In TypeScript, every `catch` receives `unknown`. The language gives you no type safety in catch blocks — you must earn it with `instanceof`. A bare `catch (e) { console.error(e) }` swallows `TypeError`, `RangeError`, and your domain errors identically. When a new error type appears, nothing warns you.
|
||||
|
||||
```typescript
|
||||
// BANNED — empty catch
|
||||
try { await fetchData() } catch {}
|
||||
try { await fetchData() } catch (e) { /* will fix later */ }
|
||||
|
||||
// BANNED — catch-and-swallow (no narrowing, no rethrow)
|
||||
try {
|
||||
const data = await api.get("/users")
|
||||
} catch (e) {
|
||||
console.error("failed", e)
|
||||
}
|
||||
|
||||
// GOOD — narrow with instanceof
|
||||
try {
|
||||
const data = await api.get("/users")
|
||||
} catch (e) {
|
||||
if (e instanceof HttpError) {
|
||||
logger.warn(`API ${e.status}: ${e.message}`)
|
||||
return fallback
|
||||
}
|
||||
throw e // unknown errors propagate
|
||||
}
|
||||
|
||||
// GOOD — top-level boundary (only place catch-all is acceptable)
|
||||
async function main(): Promise<void> { // no-excuse-ok: catch
|
||||
try {
|
||||
await run()
|
||||
} catch (e) {
|
||||
console.error("unhandled:", e)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Libraries
|
||||
|
||||
| Domain | Library | Why |
|
||||
|---|---|---|
|
||||
| HTTP framework | Hono | Lightweight, multi-runtime, middleware, OpenAPI |
|
||||
| Validation | Zod | Runtime validation + type inference |
|
||||
| ORM | Drizzle | Type-safe SQL, no codegen |
|
||||
| HTTP client | `ky` | Thin fetch wrapper (5KB); auto-throw on non-2xx, retry, timeout, hooks, prefixUrl. Browser + Node + Bun + Deno |
|
||||
| HTTP client (perf) | `undici` (direct API) | When a Node backend needs connection pooling, HTTP/2, or pipelining |
|
||||
|
||||
> **HTTP client rule** - production code must not use bare `fetch()`. It has no retry, timeout, or error-handling policy and causes silent failures during incidents. Install **`ky`** by default, and use the **`undici`** direct API when a Node backend needs high-volume requests, connection pooling, HTTP/2, or pipelining. ~~`axios`~~ is forbidden after the supply-chain compromise (2026-03). `node-fetch` is unnecessary because Node 18+ includes built-in fetch.
|
||||
| Testing | `bun test` / vitest | Fast, ESM-native |
|
||||
| Logging | `pino` | Structured JSON, fast |
|
||||
| CLI | `@clack/prompts` + `commander` | Interactive + parsing |
|
||||
|
||||
## tsconfig — the one true config
|
||||
|
||||
Scaffold a new project with all strict defaults pre-configured:
|
||||
|
||||
```bash
|
||||
bun run ../../scripts/typescript/new-project.ts my-api
|
||||
bun run ../../scripts/typescript/new-project.ts my-api --path ./projects
|
||||
```
|
||||
|
||||
Creates: `package.json` (Hono + Zod + Biome), `tsconfig.json` (ultra-strict), `biome.json`, `src/index.ts`, `.gitignore`. Works on macOS, Linux, Windows.
|
||||
|
||||
For manual setup: `bunx tsc --init`, then load `tsconfig-strict.md` for the full strict config.
|
||||
|
||||
Key flags beyond `"strict": true`:
|
||||
|
||||
| Flag | What it catches |
|
||||
|---|---|
|
||||
| `noUncheckedIndexedAccess` | `arr[0]` is `T \| undefined`, forces check |
|
||||
| `exactOptionalPropertyTypes` | `{ x?: string }` ≠ `{ x: string \| undefined }` |
|
||||
| `verbatimModuleSyntax` | Forces `import type` for type-only imports |
|
||||
| `noFallthroughCasesInSwitch` | Forgotten `break` / `return` |
|
||||
| `noPropertyAccessFromIndexSignature` | `.key` on index sig → bracket notation |
|
||||
|
||||
## Reference loading
|
||||
|
||||
Load on demand — not all at once.
|
||||
|
||||
| Need | Load |
|
||||
|---|---|
|
||||
| Strict tsconfig + Biome config | `tsconfig-strict.md` |
|
||||
| Type patterns (branded, as const, satisfies, narrowing, assertNever) | `type-patterns.md` |
|
||||
| Data modeling (type vs interface vs Zod, readonly, parse-don't-validate) | `data-modeling.md` |
|
||||
| Error handling (Result, typed errors, union vs throw) | `error-handling.md` |
|
||||
| Bootstrapping a new project (Bun, pnpm, Hono, Vite) | `bootstrap.md` |
|
||||
| Hono backend stack (hono-openapi, Scalar, Swagger) | `backend-hono.md` |
|
||||
|
||||
## No-excuse audit
|
||||
|
||||
Violations caught by `../../scripts/typescript/check-no-excuse-rules.ts`. Run after every edit session.
|
||||
|
||||
| Rule ID | Catches | Opt-out |
|
||||
|---|---|---|
|
||||
| `no-any-assertion` | `as any` | None — redesign types |
|
||||
| `no-unknown-assertion` | `as unknown` | None — redesign types |
|
||||
| `no-ts-ignore` | `@ts-ignore` | None — fix the type |
|
||||
| `no-ts-expect-error` | `@ts-expect-error` | None — fix the type |
|
||||
| `no-enum` | `enum` declarations | None — use `as const` |
|
||||
| `no-non-null-assertion` | `x!` postfix | None — narrow or `?.` |
|
||||
| `no-throw-literal` | `throw "string"` / `throw 123` | None — throw Error subclass |
|
||||
| `no-mutable-export` | `export let` / `export var` | None — use `export const` |
|
||||
| `no-any-annotation` | `: any` in parameter/return/variable types | `// no-excuse-ok: any` |
|
||||
| `no-explicit-any-return` | `(): any` or `(): Promise<any>` return types | `// no-excuse-ok: any` |
|
||||
| `empty-catch` | `catch { }` or `catch (e) { }` with empty body | `// no-excuse-ok: catch` |
|
||||
| `catch-without-narrowing` | `catch (e)` used without `instanceof` or re-throw | `// no-excuse-ok: catch` |
|
||||
|
||||
Biome enforces additional rules (noExplicitAny, noNonNullAssertion, noDefaultExport, useImportType). The script catches what Biome cannot.
|
||||
|
||||
## In tests
|
||||
|
||||
Tests are strict too, with these exceptions (configure in `biome.jsonc` overrides):
|
||||
|
||||
| In tests you may | Why |
|
||||
|---|---|
|
||||
| Use `expect()` assertions | That's how testing works |
|
||||
| Use magic numbers | Test data |
|
||||
| Access private members via bracket notation | Testing internals |
|
||||
| Skip readonly on test fixtures | Mutable setup/teardown |
|
||||
|
||||
Tests still follow the iron list — branded types, typed errors, exhaustive switch.
|
||||
|
||||
## Existing codebases
|
||||
|
||||
When editing an existing file that doesn't follow these rules: **write new code in strict style, don't refactor existing code in the same change.**
|
||||
|
||||
## Activation
|
||||
|
||||
This skill activates whenever you are writing or modifying any `.ts` or `.tsx` file. Even one-off scripts get the strict treatment.
|
||||
@@ -0,0 +1,672 @@
|
||||
# Hono Backend Stack Reference (2026)
|
||||
|
||||
> **Canonical stack**: `hono` + `hono-openapi` + `@scalar/hono-api-reference` + `@hono/swagger-ui`
|
||||
> **Runtime**: Bun (TypeScript-first)
|
||||
> **Validator**: Zod v4 (Standard Schema compliant, zero extra deps for OpenAPI)
|
||||
|
||||
---
|
||||
|
||||
## 1. Package Versions (Latest Stable)
|
||||
|
||||
| Package | Version | Source |
|
||||
|---------|---------|--------|
|
||||
| `hono` | `^4.12.5` | [peer dep of scalar](https://github.com/scalar/scalar/blob/8bcf8bf52a0da667d44eeec08648e3b1da044f97/integrations/hono/package.json#L66) |
|
||||
| `hono-openapi` | `^1.3.0` | [npm](https://registry.npmjs.org/hono-openapi) — published Mar 2, 2026 |
|
||||
| `@scalar/hono-api-reference` | `^0.10.11` | [npm](https://www.npmjs.com/package/@scalar/hono-api-reference) — published Apr 28, 2026 |
|
||||
| `@hono/swagger-ui` | `^0.6.1` | [npm](https://www.npmjs.com/package/@hono/swagger-ui) — published Apr 2026 |
|
||||
| `zod` | `^4.4.1` | [npm registry](https://registry.npmjs.org/zod) — latest stable v4 |
|
||||
|
||||
### `package.json` dependency block
|
||||
|
||||
```json
|
||||
{
|
||||
"dependencies": {
|
||||
"hono": "^4.12.5",
|
||||
"hono-openapi": "^1.3.0",
|
||||
"@scalar/hono-api-reference": "^0.10.11",
|
||||
"@hono/swagger-ui": "^0.6.1",
|
||||
"zod": "^4.4.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^5.8.0",
|
||||
"@types/bun": "latest"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
> **Peer dependencies auto-installed by `hono-openapi`**:
|
||||
> - `@hono/standard-validator@^0.2.0`
|
||||
> - `@standard-community/standard-json@^0.3.5`
|
||||
> - `@standard-community/standard-openapi@^0.2.9`
|
||||
> - `openapi-types@^12.1.3`
|
||||
>
|
||||
> [Source: `hono-openapi/package.json` peerDependencies](https://github.com/rhinobase/hono-openapi/blob/10f45a66ede3764b5e6065805fb60fd5df090466/package.json#L50-L65)
|
||||
|
||||
---
|
||||
|
||||
## 2. Complete `app.ts` — Copy-Pasteable
|
||||
|
||||
```typescript
|
||||
import { Hono } from 'hono'
|
||||
import { describeRoute, openAPIRouteHandler, resolver, validator } from 'hono-openapi'
|
||||
import { Scalar } from '@scalar/hono-api-reference'
|
||||
import { swaggerUI } from '@hono/swagger-ui'
|
||||
import { z } from 'zod'
|
||||
|
||||
// ───────────────────────────────────────────────────────────────
|
||||
// 1. Schema definitions (Zod v4 — Standard Schema native)
|
||||
// ───────────────────────────────────────────────────────────────
|
||||
|
||||
const QuerySchema = z.object({
|
||||
name: z.string().optional(),
|
||||
})
|
||||
|
||||
const ResponseSchema = z.object({
|
||||
message: z.string(),
|
||||
})
|
||||
|
||||
const JsonBodySchema = z.object({
|
||||
name: z.string(),
|
||||
age: z.number().int().min(0),
|
||||
})
|
||||
|
||||
// ───────────────────────────────────────────────────────────────
|
||||
// 2. Hono app with described routes
|
||||
// ───────────────────────────────────────────────────────────────
|
||||
|
||||
const app = new Hono()
|
||||
|
||||
// Health check (no validation)
|
||||
app.get('/health', (c) => c.json({ status: 'ok' }))
|
||||
|
||||
// A fully-documented route
|
||||
app.get(
|
||||
'/hello',
|
||||
describeRoute({
|
||||
tags: ['Greetings'],
|
||||
summary: 'Say hello',
|
||||
description: 'Returns a greeting message',
|
||||
responses: {
|
||||
200: {
|
||||
description: 'Successful greeting',
|
||||
content: {
|
||||
'application/json': {
|
||||
schema: resolver(ResponseSchema),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
validator('query', QuerySchema),
|
||||
(c) => {
|
||||
const query = c.req.valid('query')
|
||||
return c.json({ message: `Hello ${query.name ?? 'Hono'}!` })
|
||||
},
|
||||
)
|
||||
|
||||
// A POST route with JSON body validation
|
||||
app.post(
|
||||
'/users',
|
||||
describeRoute({
|
||||
tags: ['Users'],
|
||||
summary: 'Create a user',
|
||||
responses: {
|
||||
200: {
|
||||
description: 'User created',
|
||||
content: {
|
||||
'application/json': {
|
||||
schema: resolver(ResponseSchema),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
validator('json', JsonBodySchema),
|
||||
(c) => {
|
||||
const body = c.req.valid('json')
|
||||
return c.json({ message: `Created user ${body.name}` })
|
||||
},
|
||||
)
|
||||
|
||||
// ───────────────────────────────────────────────────────────────
|
||||
// 3. OpenAPI spec endpoint
|
||||
// ───────────────────────────────────────────────────────────────
|
||||
|
||||
app.get(
|
||||
'/openapi.json',
|
||||
openAPIRouteHandler(app, {
|
||||
documentation: {
|
||||
info: {
|
||||
title: 'Hono API',
|
||||
version: '1.0.0',
|
||||
description: 'Example Hono API with OpenAPI',
|
||||
},
|
||||
servers: [
|
||||
{ url: 'http://localhost:3000', description: 'Local server' },
|
||||
],
|
||||
},
|
||||
}),
|
||||
)
|
||||
|
||||
// ───────────────────────────────────────────────────────────────
|
||||
// 4. Scalar API Reference UI
|
||||
// ───────────────────────────────────────────────────────────────
|
||||
|
||||
app.get(
|
||||
'/scalar',
|
||||
Scalar({
|
||||
url: '/openapi.json',
|
||||
theme: 'saturn',
|
||||
pageTitle: 'Hono API Reference',
|
||||
}),
|
||||
)
|
||||
|
||||
// ───────────────────────────────────────────────────────────────
|
||||
// 5. Swagger UI (parallel mount)
|
||||
// ───────────────────────────────────────────────────────────────
|
||||
|
||||
app.get(
|
||||
'/swagger',
|
||||
swaggerUI({
|
||||
url: '/openapi.json',
|
||||
title: 'Swagger UI',
|
||||
}),
|
||||
)
|
||||
|
||||
// ───────────────────────────────────────────────────────────────
|
||||
// 6. Bun canonical entrypoint
|
||||
// ───────────────────────────────────────────────────────────────
|
||||
|
||||
export default app
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. `hono-openapi` API Reference
|
||||
|
||||
### Import paths
|
||||
|
||||
**There is only one import path.** `hono-openapi` exports everything from its root:
|
||||
|
||||
```typescript
|
||||
import {
|
||||
describeRoute, // middleware to annotate a route with OpenAPI metadata
|
||||
describeResponse, // attach response schemas directly to a handler
|
||||
validator, // validation middleware (wraps @hono/standard-validator)
|
||||
resolver, // wrap a Zod/Valibot/etc schema for OpenAPI responses
|
||||
openAPIRouteHandler, // serve the generated OpenAPI JSON document
|
||||
generateSpecs, // programmatically generate the spec (for build-time caching)
|
||||
} from 'hono-openapi'
|
||||
```
|
||||
|
||||
**Evidence** ([`src/index.ts`](https://github.com/rhinobase/hono-openapi/blob/10f45a66ede3764b5e6065805fb60fd5df090466/src/index.ts#L1-L9)):
|
||||
|
||||
```typescript
|
||||
export { generateSpecs, openAPIRouteHandler } from "./handler.js";
|
||||
export {
|
||||
describeResponse,
|
||||
describeRoute,
|
||||
loadVendor,
|
||||
resolver,
|
||||
validator,
|
||||
} from "./middlewares.js";
|
||||
```
|
||||
|
||||
> **No subpath exports** such as `hono-openapi/zod` or `hono-openapi/valibot`. The package uses Standard Schema and auto-detects the validator vendor.
|
||||
|
||||
### `describeRoute()` middleware
|
||||
|
||||
Attach OpenAPI metadata to any Hono route. Use `resolver()` for response body schemas.
|
||||
|
||||
```typescript
|
||||
app.get(
|
||||
'/path',
|
||||
describeRoute({
|
||||
tags: ['Users'],
|
||||
summary: 'Get user',
|
||||
description: 'Retrieve a single user by ID',
|
||||
responses: {
|
||||
200: {
|
||||
description: 'User found',
|
||||
content: {
|
||||
'application/json': {
|
||||
schema: resolver(UserSchema),
|
||||
},
|
||||
},
|
||||
},
|
||||
404: {
|
||||
description: 'User not found',
|
||||
},
|
||||
},
|
||||
}),
|
||||
handler,
|
||||
)
|
||||
```
|
||||
|
||||
**Evidence** ([`src/middlewares.ts`](https://github.com/rhinobase/hono-openapi/blob/10f45a66ede3764b5e6065805fb60fd5df090466/src/middlewares.ts#L244-L254)):
|
||||
|
||||
```typescript
|
||||
export function describeRoute(spec: DescribeRouteOptions): MiddlewareHandler {
|
||||
const middleware: MiddlewareHandler = async (_c, next) => {
|
||||
await next();
|
||||
};
|
||||
return Object.assign(middleware, {
|
||||
[uniqueSymbol]: { spec },
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
### `validator()` middleware
|
||||
|
||||
Validates `query`, `json`, `param`, or `form` and **automatically** injects the request schema into the OpenAPI document. No manual `request` block in `describeRoute()` is required.
|
||||
|
||||
```typescript
|
||||
validator('query', QuerySchema) // ?name=foo
|
||||
validator('json', JsonBodySchema) // POST body
|
||||
validator('param', ParamSchema) // /users/:id
|
||||
validator('form', FormSchema) // multipart/form-data
|
||||
```
|
||||
|
||||
**Evidence** ([`src/middlewares.ts`](https://github.com/rhinobase/hono-openapi/blob/10f45a66ede3764b5e6065805fb60fd5df090466/src/middlewares.ts#L199-L237)):
|
||||
|
||||
```typescript
|
||||
export function validator<Schema extends StandardSchemaV1, ...>(
|
||||
target: Target,
|
||||
schema: Schema,
|
||||
hook?: Hook<...>,
|
||||
options?: ResolverReturnType["options"],
|
||||
): MiddlewareHandler<E, P, V> {
|
||||
const middleware = sValidator(target, schema, hook);
|
||||
return Object.assign(middleware, {
|
||||
[uniqueSymbol]: { target, ...resolver(schema, options), options },
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
### `openAPIRouteHandler()` — serving the spec
|
||||
|
||||
```typescript
|
||||
app.get(
|
||||
'/openapi.json',
|
||||
openAPIRouteHandler(app, {
|
||||
documentation: {
|
||||
info: { title: 'Hono API', version: '1.0.0' },
|
||||
servers: [{ url: 'http://localhost:3000' }],
|
||||
},
|
||||
}),
|
||||
)
|
||||
```
|
||||
|
||||
**Evidence** ([`src/handler.ts`](https://github.com/rhinobase/hono-openapi/blob/10f45a66ede3764b5e6065805fb60fd5df090466/src/handler.ts#L42-L59)):
|
||||
|
||||
```typescript
|
||||
export function openAPIRouteHandler<...>(
|
||||
hono: Hono<E, S, P>,
|
||||
options?: Partial<GenerateSpecOptions>,
|
||||
): MiddlewareHandler<E, P, I> {
|
||||
let specs: OpenAPIV3_1.Document;
|
||||
return async (c) => {
|
||||
if (specs) return c.json(specs);
|
||||
specs = await generateSpecs(hono, options, c);
|
||||
return c.json(specs);
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
> **Mount path convention**: `/openapi.json` is the most common. Some projects use `/openapi/spec.json` (e.g. [NamesMT/starter-monorepo](https://github.com/NamesMT/starter-monorepo/blob/main/apps/backend/src/openAPI.ts)).
|
||||
|
||||
---
|
||||
|
||||
## 4. `@scalar/hono-api-reference` Setup
|
||||
|
||||
### Import path and package name
|
||||
|
||||
```typescript
|
||||
import { Scalar } from '@scalar/hono-api-reference'
|
||||
```
|
||||
|
||||
> **Deprecated**: `apiReference` is still exported but deprecated in favor of `Scalar` ([PR #5297](https://github.com/scalar/scalar/pull/5297)).
|
||||
|
||||
**Evidence** ([`integrations/hono/src/index.ts`](https://github.com/scalar/scalar/blob/8bcf8bf52a0da667d44eeec08648e3b1da044f97/integrations/hono/src/index.ts#L1-L9)):
|
||||
|
||||
```typescript
|
||||
import { Scalar } from './scalar'
|
||||
export {
|
||||
Scalar,
|
||||
/**
|
||||
* @deprecated Use `Scalar` instead.
|
||||
*/
|
||||
Scalar as apiReference,
|
||||
}
|
||||
```
|
||||
|
||||
### Mount path convention
|
||||
|
||||
Common choices:
|
||||
- `/scalar` — matches the middleware name
|
||||
- `/docs` — generic documentation endpoint
|
||||
- `/openapi/ui` — nested under the OpenAPI prefix
|
||||
|
||||
### Configuration options
|
||||
|
||||
The Hono middleware accepts the **universal Scalar configuration** plus Hono-specific overrides (`pageTitle`, `cdn`).
|
||||
|
||||
```typescript
|
||||
app.get('/scalar', Scalar({
|
||||
// ── Source (required) ──
|
||||
url: '/openapi.json', // URL to the OpenAPI spec
|
||||
|
||||
// ── Appearance ──
|
||||
theme: 'saturn', // 'alternate' | 'default' | 'moon' | 'purple'
|
||||
// | 'solarized' | 'bluePlanet' | 'deepSpace'
|
||||
// | 'saturn' | 'kepler' | 'elysiajs' | 'fastify'
|
||||
// | 'mars' | 'laserwave' | 'none'
|
||||
pageTitle: 'My API Docs', // HTML <title>
|
||||
customCss: '.sidebar { ... }', // injected <style> block
|
||||
metaData: { title: '...' }, // SEO meta tags (unhead format)
|
||||
favicon: '/favicon.svg',
|
||||
|
||||
// ── Behavior ──
|
||||
layout: 'modern', // 'modern' | 'classic'
|
||||
darkMode: true,
|
||||
forceDarkModeState: 'dark', // 'dark' | 'light'
|
||||
hideDarkModeToggle: false,
|
||||
hideModels: false,
|
||||
hideSearch: false,
|
||||
hideTestRequestButton: false,
|
||||
showOperationId: false,
|
||||
showSidebar: true,
|
||||
|
||||
// ── Proxy / Server ──
|
||||
proxyUrl: 'https://proxy.scalar.com',
|
||||
baseServerURL: 'http://localhost:3000',
|
||||
servers: [{ url: 'http://localhost:3000' }],
|
||||
|
||||
// ── CDN ──
|
||||
cdn: 'https://cdn.jsdelivr.net/npm/@scalar/api-reference',
|
||||
|
||||
// ── Advanced ──
|
||||
authentication: { ... },
|
||||
hiddenClients: ['unirest'],
|
||||
defaultHttpClient: { targetKey: 'js', clientKey: 'fetch' },
|
||||
plugins: [...],
|
||||
pathRouting: { basePath: '/reference' },
|
||||
mcp: { name: 'My MCP', url: '...' },
|
||||
}))
|
||||
```
|
||||
|
||||
**Evidence** — Scalar types define the full schema:
|
||||
- [Base configuration (themes, proxy, etc.)](https://github.com/scalar/scalar/blob/8bcf8bf52a0da667d44eeec08648e3b1da044f97/packages/types/src/api-reference/base-configuration.ts#L110-L129)
|
||||
- [HTML rendering configuration (`pageTitle`, `cdn`)](https://github.com/scalar/scalar/blob/8bcf8bf52a0da667d44eeec08648e3b1da044f97/packages/types/src/api-reference/html-rendering-configuration.ts#L8-L23)
|
||||
- [Source configuration (`url`, `content`)](https://github.com/scalar/scalar/blob/8bcf8bf52a0da667d44eeec08648e3b1da044f97/packages/types/src/api-reference/source-configuration.ts#L8-L55)
|
||||
- [Full API reference configuration](https://github.com/scalar/scalar/blob/8bcf8bf52a0da667d44eeec08648e3b1da044f97/packages/types/src/api-reference/api-reference-configuration.ts#L22-L379)
|
||||
|
||||
### Dynamic configuration (request-aware)
|
||||
|
||||
```typescript
|
||||
app.get('/scalar', Scalar((c) => ({
|
||||
url: '/openapi.json',
|
||||
proxyUrl: c.env.ENVIRONMENT === 'development'
|
||||
? 'https://proxy.scalar.com'
|
||||
: undefined,
|
||||
})))
|
||||
```
|
||||
|
||||
**Evidence** ([`integrations/hono/src/scalar.ts`](https://github.com/scalar/scalar/blob/8bcf8bf52a0da667d44eeec08648e3b1da044f97/integrations/hono/src/scalar.ts#L75-L94)):
|
||||
|
||||
```typescript
|
||||
export const Scalar = <E extends Env>(configOrResolver: Configuration<E>): MiddlewareHandler<E> => {
|
||||
return async (c) => {
|
||||
let resolvedConfig: Partial<ApiReferenceConfiguration> = {}
|
||||
if (typeof configOrResolver === 'function') {
|
||||
resolvedConfig = await configOrResolver(c)
|
||||
} else {
|
||||
resolvedConfig = configOrResolver
|
||||
}
|
||||
// ...
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. `@hono/swagger-ui` Setup
|
||||
|
||||
### Import path and package name
|
||||
|
||||
```typescript
|
||||
import { swaggerUI } from '@hono/swagger-ui'
|
||||
```
|
||||
|
||||
**Evidence** ([`packages/swagger-ui/src/index.ts`](https://github.com/honojs/middleware/blob/eb443a2fbda674bbe12d3f30e96854bb0cad6232/packages/swagger-ui/src/index.ts#L93)):
|
||||
|
||||
```typescript
|
||||
export { middleware as swaggerUI, SwaggerUI }
|
||||
```
|
||||
|
||||
### Mount path convention
|
||||
|
||||
Common choices:
|
||||
- `/swagger` — explicit
|
||||
- `/ui` — used in Hono official examples
|
||||
- `/docs` — generic
|
||||
|
||||
### Configuration options
|
||||
|
||||
```typescript
|
||||
app.get('/swagger', swaggerUI({
|
||||
url: '/openapi.json', // URL to the OpenAPI spec (required)
|
||||
title: 'Swagger UI', // HTML page title
|
||||
version: 'latest', // Swagger UI CDN version
|
||||
// Any standard Swagger UI option also works:
|
||||
// presets, plugins, urls, etc.
|
||||
}))
|
||||
```
|
||||
|
||||
**Evidence** ([`packages/swagger-ui/src/index.ts`](https://github.com/honojs/middleware/blob/eb443a2fbda674bbe12d3f30e96854bb0cad6232/packages/swagger-ui/src/index.ts#L8-L43)):
|
||||
|
||||
```typescript
|
||||
type SwaggerUIOptions = OriginalSwaggerUIOptions & DistSwaggerUIOptions
|
||||
|
||||
const middleware = <E extends Env>(options: SwaggerUIOptions): MiddlewareHandler<E> =>
|
||||
async (c) => {
|
||||
const title = options?.title ?? 'SwaggerUI'
|
||||
return c.html(/* html */ `...`)
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Bun Runtime Entrypoint
|
||||
|
||||
### Canonical shape for `bun run`
|
||||
|
||||
```typescript
|
||||
import { Hono } from 'hono'
|
||||
|
||||
const app = new Hono()
|
||||
// ... routes ...
|
||||
|
||||
export default app
|
||||
```
|
||||
|
||||
**Evidence** ([Hono Bun docs](https://hono.dev/docs/getting-started/bun)):
|
||||
|
||||
> ```ts
|
||||
> import { Hono } from 'hono'
|
||||
> const app = new Hono()
|
||||
> app.get('/', (c) => c.text('Hello Bun!'))
|
||||
> export default app
|
||||
> ```
|
||||
|
||||
### Custom port
|
||||
|
||||
```typescript
|
||||
export default {
|
||||
port: 3000,
|
||||
fetch: app.fetch,
|
||||
}
|
||||
```
|
||||
|
||||
**Evidence** ([Hono Bun docs — Change port number](https://hono.dev/docs/getting-started/bun)):
|
||||
|
||||
> ```ts
|
||||
> export default {
|
||||
> port: 3000,
|
||||
> fetch: app.fetch,
|
||||
> }
|
||||
> ```
|
||||
|
||||
### `package.json` scripts for Bun
|
||||
|
||||
```json
|
||||
{
|
||||
"scripts": {
|
||||
"dev": "bun run --hot src/index.ts",
|
||||
"start": "bun run src/index.ts",
|
||||
"build": "tsc --noEmit"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. OpenAPI Version
|
||||
|
||||
### `hono-openapi` emits **OpenAPI 3.1.0** by default
|
||||
|
||||
This is **hardcoded** in the source and **not configurable** at runtime:
|
||||
|
||||
**Evidence** ([`src/handler.ts` line 120](https://github.com/rhinobase/hono-openapi/blob/10f45a66ede3764b5e6065805fb60fd5df090466/src/handler.ts#L120)):
|
||||
|
||||
```typescript
|
||||
return {
|
||||
openapi: "3.1.0",
|
||||
..._documentation,
|
||||
// ...
|
||||
} satisfies OpenAPIV3_1.Document;
|
||||
```
|
||||
|
||||
> If you need OpenAPI 3.0.x, you must post-process the generated spec or use `@hono/zod-openapi` (the older package) instead. The user explicitly requested `hono-openapi`, so document that 3.1.0 is the only output.
|
||||
|
||||
---
|
||||
|
||||
## 8. Zod v3 vs Zod v4
|
||||
|
||||
| Feature | Zod v3 | Zod v4 |
|
||||
|---------|--------|--------|
|
||||
| Standard Schema | ❌ No | ✅ Yes (native) |
|
||||
| `hono-openapi` extra deps | `zod-openapi@4` | None |
|
||||
| Import path | `import { z } from 'zod'` | `import { z } from 'zod'` (or `zod/v4` for explicit) |
|
||||
|
||||
**For Zod v3 users**, install the compatibility layer:
|
||||
|
||||
```bash
|
||||
npm install zod-openapi@4
|
||||
```
|
||||
|
||||
Then use `zod-openapi`'s `.openapi()` for metadata and `.meta({ ref: 'Name' })` for component references. `hono-openapi`'s `resolver()` will still work, but the underlying schema conversion relies on `zod-openapi@4`.
|
||||
|
||||
**Evidence** ([HonoHub Zod docs](https://honohub.dev/docs/openapi/zod)):
|
||||
|
||||
> "For zod v3, you can use the `zod-openapi` library. You need to install `zod-openapi@4` for this to work properly."
|
||||
|
||||
**For Zod v4 users** (recommended in 2026), no extra packages are needed. `z.date()` is automatically converted to `{ type: 'string', format: 'date-time' }`.
|
||||
|
||||
**Evidence** ([`src/middlewares.ts` Zod v4 date override](https://github.com/rhinobase/hono-openapi/blob/10f45a66ede3764b5e6065805fb60fd5df090466/src/middlewares.ts#L63-L71)):
|
||||
|
||||
```typescript
|
||||
const zodV4DateOverride = (ctx: { ... }) => {
|
||||
if (ctx.zodSchema._zod.def.type === "date") {
|
||||
ctx.jsonSchema.type = "string";
|
||||
ctx.jsonSchema.format = "date-time";
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9. Real-World Example
|
||||
|
||||
**NamesMT/starter-monorepo** — a public monorepo starter using `hono-openapi` + `@scalar/hono-api-reference` together:
|
||||
|
||||
- File: [`apps/backend/src/openAPI.ts`](https://github.com/NamesMT/starter-monorepo/blob/main/apps/backend/src/openAPI.ts)
|
||||
- Pattern: mounts spec at `/openapi/spec.json` and Scalar UI at `/openapi/ui`
|
||||
|
||||
```typescript
|
||||
import type { Hono } from 'hono'
|
||||
import { Scalar } from '@scalar/hono-api-reference'
|
||||
import { openAPIRouteHandler } from 'hono-openapi'
|
||||
|
||||
export function setupOpenAPI(app: Hono<any, any>, prefix = '/openapi') {
|
||||
app.get(
|
||||
`${prefix}/spec.json`,
|
||||
openAPIRouteHandler(app, {
|
||||
documentation: {
|
||||
info: {
|
||||
title: `starter-monorepo's backend`,
|
||||
version: '1.0.0',
|
||||
description: 'My amazing API',
|
||||
},
|
||||
},
|
||||
}),
|
||||
)
|
||||
|
||||
app.get(
|
||||
`${prefix}/ui`,
|
||||
Scalar({
|
||||
theme: 'deepSpace',
|
||||
url: `${prefix}/spec.json`,
|
||||
}),
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
> **Note**: No public repo was found using all four (`hono-openapi` + `Scalar` + `swagger-ui` + `hono`) in a single file. The canonical combination in the wild is `hono-openapi` + `Scalar`. Adding `swagger-ui` is a trivial parallel mount (shown in the `app.ts` above).
|
||||
|
||||
---
|
||||
|
||||
## 10. Common Pitfalls
|
||||
|
||||
1. **Using `openAPISpecs` instead of `openAPIRouteHandler`**
|
||||
Some docs (e.g. HONC) use `openAPISpecs` — this is **not** the current export name. The correct function is `openAPIRouteHandler` ([source](https://github.com/rhinobase/hono-openapi/blob/10f45a66ede3764b5e6065805fb60fd5df090466/src/index.ts#L1)).
|
||||
|
||||
2. **Importing from `hono-openapi/zod`**
|
||||
There are **no subpath exports**. Always import from `hono-openapi` directly.
|
||||
|
||||
3. **Forgetting `@hono/standard-validator`**
|
||||
It is a peer dependency of `hono-openapi`. Modern package managers (npm ≥ 7, pnpm, bun) auto-install it. If you see validation errors, ensure it is present in `node_modules`.
|
||||
|
||||
4. **Using `@hono/zod-openapi` (the OLD package)**
|
||||
The user explicitly wants `hono-openapi` (the newer, middleware-based, Standard Schema package). Do not confuse with `@hono/zod-openapi` which wraps the `Hono` class into `OpenAPIHono`.
|
||||
|
||||
5. **Swagger UI `spec` option**
|
||||
`@hono/swagger-ui` does **not** accept a `spec` option to embed the document directly. It only accepts `url` (or `urls`) pointing to an external spec endpoint. If you need embedded specs, use Scalar's `content` option instead.
|
||||
|
||||
---
|
||||
|
||||
## 11. Quick Start Commands
|
||||
|
||||
```bash
|
||||
# 1. Create project
|
||||
mkdir my-api && cd my-api
|
||||
bun init -y
|
||||
|
||||
# 2. Install dependencies
|
||||
bun add hono hono-openapi @scalar/hono-api-reference @hono/swagger-ui zod
|
||||
|
||||
# 3. Add TypeScript
|
||||
bun add -d typescript @types/bun
|
||||
|
||||
# 4. Write app.ts (copy from section 2 above)
|
||||
# 5. Run
|
||||
bun run --hot app.ts
|
||||
```
|
||||
|
||||
Endpoints after startup:
|
||||
- `GET /health` — health check
|
||||
- `GET /hello?name=world` — documented route
|
||||
- `POST /users` — validated JSON body route
|
||||
- `GET /openapi.json` — raw OpenAPI 3.1.0 spec
|
||||
- `GET /scalar` — Scalar API Reference UI
|
||||
- `GET /swagger` — Swagger UI
|
||||
@@ -0,0 +1,199 @@
|
||||
# Bootstrap — Runtime, Package Manager, Tooling
|
||||
|
||||
When starting a new TypeScript project (or scripting against the world), the choice of runtime, package manager, framework, and toolchain compounds. The wrong default at minute zero costs hours every week. The right defaults for 2026:
|
||||
|
||||
## Runtime decision tree
|
||||
|
||||
```
|
||||
Is this a CLI / script / single-binary tool?
|
||||
└─ Yes → Bun (single executable, hot reload, native TS)
|
||||
Use `bun run script.ts` directly. No build step.
|
||||
|
||||
Is this a backend service?
|
||||
├─ Edge (Cloudflare Workers / Vercel / Deno Deploy) → match the platform
|
||||
├─ Bun-supported runtime → Bun + Hono
|
||||
├─ Need Node-only deps (sharp, native modules without Bun support) → Node + Hono
|
||||
└─ Otherwise → Bun + Hono
|
||||
|
||||
Is this a frontend?
|
||||
└─ Vite (regardless of framework). Bun for the package manager.
|
||||
|
||||
Is this a library to publish to npm?
|
||||
└─ tsdown (or unbuild). Targets Node 20+. Use pnpm for monorepo workspaces.
|
||||
```
|
||||
|
||||
## Bun is the default runtime
|
||||
|
||||
Use Bun for:
|
||||
- Scripts and CLIs (`bun run` is faster than `tsx` and `ts-node`)
|
||||
- New backends (Hono runs natively, hot reload via `bun --hot`)
|
||||
- Test runner (`bun test` is built-in, faster than vitest for small suites)
|
||||
- Package manager (`bun install` is faster than `pnpm` and far faster than `npm`)
|
||||
|
||||
Use Node when:
|
||||
- A dependency uses native modules Bun can't load (rare in 2026; check the dep's release notes)
|
||||
- Production target is a Node-specific platform (some serverless platforms don't run Bun yet)
|
||||
- You're contributing to a Node-only project
|
||||
|
||||
`bunx` replaces `npx`. `bun create` scaffolds projects.
|
||||
|
||||
## Package manager — pnpm > npm
|
||||
|
||||
If you must use Node, use pnpm. NEVER npm except in legacy projects you don't control.
|
||||
|
||||
Why pnpm:
|
||||
- Content-addressable store: 10x less disk usage on a machine with many projects
|
||||
- Strict node_modules layout: phantom dependencies fail at install time, not at runtime
|
||||
- Workspaces are first-class
|
||||
- Significantly faster than npm
|
||||
|
||||
Why not yarn:
|
||||
- Yarn classic is unmaintained
|
||||
- Yarn berry's "PnP" mode breaks with editor tooling more often than it should
|
||||
- pnpm has caught up on every yarn berry feature people actually use
|
||||
|
||||
Why not npm:
|
||||
- Slowest of the three
|
||||
- No proper workspace story until very recently
|
||||
- Phantom dependencies allowed by default
|
||||
|
||||
```bash
|
||||
# Convert npm/yarn → pnpm
|
||||
pnpm import # reads package-lock.json or yarn.lock and produces pnpm-lock.yaml
|
||||
rm -rf node_modules package-lock.json yarn.lock
|
||||
pnpm install
|
||||
```
|
||||
|
||||
## Backend framework — Hono
|
||||
|
||||
Use Hono for any new HTTP service. It is:
|
||||
- Type-safe end-to-end (request/response types flow through middleware)
|
||||
- Edge-compatible (runs on Bun, Node, Cloudflare Workers, Deno, AWS Lambda)
|
||||
- Faster than Express, Fastify, and most of its peers in synthetic benchmarks
|
||||
- Maintained, opinionated, and documented well
|
||||
|
||||
When Hono → ALWAYS pair with `hono-openapi` + `@scalar/hono-api-reference` + `@hono/swagger-ui`. Full setup with copy-pasteable `app.ts`: [backend-hono.md](backend-hono.md).
|
||||
|
||||
NEVER:
|
||||
- Express for new services. Express is the COBOL of Node — works, but writes itself out of every benchmark.
|
||||
- Fastify for new services. Hono ships with better TypeScript ergonomics.
|
||||
- NestJS for new services. The Angular-flavoured DI/decorator stack is overkill for ~95% of services.
|
||||
- Bare `Bun.serve` or `node:http` unless you have a specific reason. Lose middleware, routing, validation. Reinvent everything.
|
||||
|
||||
## Frontend tooling — Vite
|
||||
|
||||
Vite for any frontend. Replaces webpack, parcel, rollup-as-app-bundler. Works with React, Vue, Svelte, Solid, Preact, vanilla.
|
||||
|
||||
```bash
|
||||
bun create vite my-app -- --template react-ts
|
||||
cd my-app
|
||||
bun install
|
||||
bun run dev
|
||||
```
|
||||
|
||||
## Lint + format — Biome
|
||||
|
||||
Biome replaces ESLint + Prettier with one tool, written in Rust, ~30x faster.
|
||||
|
||||
```bash
|
||||
bun add --dev @biomejs/biome
|
||||
bun biome init
|
||||
```
|
||||
|
||||
`biome.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"$schema": "https://biomejs.dev/schemas/1.9.0/schema.json",
|
||||
"organizeImports": { "enabled": true },
|
||||
"linter": { "enabled": true, "rules": { "recommended": true } },
|
||||
"formatter": { "enabled": true, "indentStyle": "space", "indentWidth": 2 }
|
||||
}
|
||||
```
|
||||
|
||||
Use ESLint only when:
|
||||
- You have an ESLint plugin Biome doesn't replicate (rare in 2026)
|
||||
- You're contributing to an existing ESLint project
|
||||
|
||||
Never run both — pick one.
|
||||
|
||||
## Test runner — bun test or vitest
|
||||
|
||||
| Runner | Use when |
|
||||
|---|---|
|
||||
| `bun test` | Bun project, simple unit tests, no TypeScript path aliases that need vite-style resolution |
|
||||
| `vitest` | Vite-based frontend, complex test infrastructure (DOM testing, snapshot, in-browser tests), or you need vitest-specific features |
|
||||
|
||||
NEVER Jest for a new project. Jest's CommonJS-first design fights every modern Node/TS project.
|
||||
|
||||
## TypeScript
|
||||
|
||||
`tsconfig.json` for a Bun + Hono backend:
|
||||
|
||||
```json
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"lib": ["ESNext"],
|
||||
"types": ["bun-types"],
|
||||
"strict": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noImplicitReturns": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"exactOptionalPropertyTypes": true,
|
||||
"esModuleInterop": true,
|
||||
"isolatedModules": true,
|
||||
"resolveJsonModule": true,
|
||||
"skipLibCheck": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"noEmit": true
|
||||
},
|
||||
"include": ["src/**/*", "tests/**/*"]
|
||||
}
|
||||
```
|
||||
|
||||
`verbatimModuleSyntax: true` enforces explicit `import type { ... }` for type-only imports — pairs with the no-excuse rule on type-only imports.
|
||||
|
||||
`noEmit: true` because `bun run` and `bun build` handle compilation. The `tsc` command becomes a typechecker only.
|
||||
|
||||
## Quick-start: Bun + Hono backend
|
||||
|
||||
```bash
|
||||
mkdir my-api && cd my-api
|
||||
bun init -y
|
||||
bun add hono hono-openapi @scalar/hono-api-reference @hono/swagger-ui zod
|
||||
bun add --dev @biomejs/biome typescript
|
||||
bun biome init
|
||||
```
|
||||
|
||||
`package.json` scripts:
|
||||
|
||||
```json
|
||||
{
|
||||
"scripts": {
|
||||
"dev": "bun run --hot src/index.ts",
|
||||
"start": "bun run src/index.ts",
|
||||
"build": "bun build src/index.ts --target bun --outdir dist",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"lint": "biome check --write src tests",
|
||||
"test": "bun test"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Wire the `app.ts` from [backend-hono.md](backend-hono.md). You have a documented, validated, OpenAPI-spec-emitting service in ~15 minutes.
|
||||
|
||||
## When NOT to bootstrap from scratch
|
||||
|
||||
| Situation | Use |
|
||||
|---|---|
|
||||
| Internal tool with auth/admin/dashboards | Next.js (full-stack) - lots of free wiring |
|
||||
| Documentation site | Astro or VitePress |
|
||||
| Real-time features (WebRTC, complex sockets) | Bun + Hono + a real-time library |
|
||||
| Data-heavy SPA | Vite + React + TanStack Query + TanStack Router |
|
||||
|
||||
For greenfield backend services, Bun + Hono. Always.
|
||||
@@ -0,0 +1,202 @@
|
||||
# Data Modeling
|
||||
|
||||
Which construct to use, how to structure data, and why readonly is the default.
|
||||
|
||||
---
|
||||
|
||||
## Decision flowchart
|
||||
|
||||
```
|
||||
Is it a fixed set of named constants?
|
||||
YES → as const object + literal union type
|
||||
NO ↓
|
||||
Is it just branding a primitive (string, number)?
|
||||
YES → Branded type
|
||||
NO ↓
|
||||
Is it an interface / contract?
|
||||
YES → interface (structural typing is the default in TS)
|
||||
NO ↓
|
||||
Does the data cross a trust boundary (user input, API, file)?
|
||||
YES → Zod schema + z.infer<typeof schema>
|
||||
NO ↓
|
||||
Is it a union of possible outcomes?
|
||||
YES → Discriminated union (kind/type field)
|
||||
NO ↓
|
||||
Is it structured data with named fields?
|
||||
YES → type alias with readonly properties
|
||||
NO → you probably don't need a new type
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Container reference
|
||||
|
||||
### type alias — internal data
|
||||
|
||||
The default for structured data inside your codebase. Zero runtime cost.
|
||||
|
||||
```typescript
|
||||
type User = {
|
||||
readonly id: UserId
|
||||
readonly name: string
|
||||
readonly email: string
|
||||
}
|
||||
|
||||
type Point = {
|
||||
readonly x: number
|
||||
readonly y: number
|
||||
}
|
||||
```
|
||||
|
||||
All properties `readonly`. Mutable only when mutation is the documented purpose.
|
||||
|
||||
### interface — contracts and extension
|
||||
|
||||
Use when you need declaration merging or `extends`.
|
||||
|
||||
```typescript
|
||||
interface Repository<T> {
|
||||
get(id: string): Promise<T | null>
|
||||
save(entity: T): Promise<void>
|
||||
}
|
||||
|
||||
interface UserRepository extends Repository<User> {
|
||||
findByEmail(email: string): Promise<User | null>
|
||||
}
|
||||
```
|
||||
|
||||
### interface vs type — when to use which
|
||||
|
||||
| Use | When |
|
||||
|---|---|
|
||||
| `type` | Union types, intersections, mapped types, utility types, internal data shapes |
|
||||
| `interface` | Contracts that will be `implements`ed or `extends`ed, declaration merging needed |
|
||||
| **Default** | **`type` — unless you have a specific reason for `interface`** |
|
||||
|
||||
### Zod schema — trust boundary guardian
|
||||
|
||||
Use when data enters your system. Validates at runtime, infers types at compile time.
|
||||
|
||||
```typescript
|
||||
import { z } from "zod"
|
||||
|
||||
const CreateUserSchema = z.object({
|
||||
name: z.string().min(1),
|
||||
email: z.string().email(),
|
||||
age: z.number().int().min(0),
|
||||
})
|
||||
type CreateUser = z.infer<typeof CreateUserSchema>
|
||||
|
||||
const UserResponseSchema = z.object({
|
||||
id: z.string().uuid(),
|
||||
name: z.string(),
|
||||
email: z.string(),
|
||||
})
|
||||
type UserResponse = z.infer<typeof UserResponseSchema>
|
||||
```
|
||||
|
||||
**The one rule**: data crosses a trust boundary → Zod. Everything else → plain type/interface.
|
||||
Never use Zod for internal-only data. The runtime validation cost and Zod coupling are unnecessary.
|
||||
|
||||
### as const — fixed constants
|
||||
|
||||
Replaces `enum` entirely. Type-safe, tree-shakeable, no runtime overhead.
|
||||
|
||||
```typescript
|
||||
const ROLES = ["admin", "user", "guest"] as const
|
||||
type Role = (typeof ROLES)[number]
|
||||
|
||||
const STATUS = {
|
||||
ACTIVE: "active",
|
||||
INACTIVE: "inactive",
|
||||
DELETED: "deleted",
|
||||
} as const
|
||||
type Status = (typeof STATUS)[keyof typeof STATUS]
|
||||
```
|
||||
|
||||
### Discriminated union — multiple outcomes
|
||||
|
||||
```typescript
|
||||
type GetUserResult =
|
||||
| { readonly kind: "found"; readonly user: User }
|
||||
| { readonly kind: "not_found"; readonly id: UserId }
|
||||
| { readonly kind: "forbidden"; readonly reason: string }
|
||||
```
|
||||
|
||||
Each variant has a `kind` discriminant. TypeScript narrows on `switch (result.kind)`.
|
||||
|
||||
---
|
||||
|
||||
## Quick lookup
|
||||
|
||||
| Situation | Use |
|
||||
|---|---|
|
||||
| User input, API request/response | Zod schema + `z.infer` |
|
||||
| Internal value object | `type` with `readonly` properties |
|
||||
| Function with multiple outcomes | Discriminated union |
|
||||
| Contract for implementations | `interface` |
|
||||
| Fixed constants | `as const` + literal union |
|
||||
| Distinct primitive (UserId vs OrderId) | Branded type |
|
||||
| Dict shape / key-value map | `Record<K, V>` or index signature |
|
||||
|
||||
---
|
||||
|
||||
## Readonly by default
|
||||
|
||||
Every property is `readonly` unless mutation is the documented purpose.
|
||||
|
||||
```typescript
|
||||
// DEFAULT — readonly
|
||||
type Config = {
|
||||
readonly apiUrl: string
|
||||
readonly timeout: number
|
||||
}
|
||||
|
||||
// Arrays too
|
||||
function getUsers(): readonly User[] { ... }
|
||||
|
||||
// Utility for existing types
|
||||
type ReadonlyUser = Readonly<User>
|
||||
type DeepReadonlyConfig = Readonly<Config>
|
||||
```
|
||||
|
||||
For mutable state (rare), document why:
|
||||
|
||||
```typescript
|
||||
/** Counter state — mutation is the entire purpose. */
|
||||
type CounterState = {
|
||||
count: number // intentionally mutable
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Parse, don't validate
|
||||
|
||||
Validate at the boundary. Inside the boundary, types are proof of validity.
|
||||
|
||||
```typescript
|
||||
// BAD — validate then pass raw data
|
||||
function processEmail(email: string): void {
|
||||
if (!email.includes("@")) throw new Error("invalid")
|
||||
// still a raw string downstream
|
||||
}
|
||||
|
||||
// GOOD — parse into typed value at boundary
|
||||
const EmailSchema = z.string().email().brand("Email")
|
||||
type Email = z.infer<typeof EmailSchema>
|
||||
|
||||
function sendWelcome(email: Email): void { ... }
|
||||
|
||||
// Boundary code
|
||||
const parsed = EmailSchema.parse(rawInput) // Email or throws
|
||||
sendWelcome(parsed) // no re-validation needed
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- TypeScript Handbook: [Object Types](https://www.typescriptlang.org/docs/handbook/2/objects.html)
|
||||
- Zod: [docs](https://zod.dev)
|
||||
- Total TypeScript: [Type vs Interface](https://www.totaltypescript.com/type-vs-interface-which-should-you-use)
|
||||
+169
@@ -0,0 +1,169 @@
|
||||
# Error Handling
|
||||
|
||||
Typed errors, exhaustive matching, Result pattern, and resource safety.
|
||||
|
||||
---
|
||||
|
||||
## Typed errors — no bare strings
|
||||
|
||||
Error classes carry structured data. Callers know exactly what can go wrong.
|
||||
|
||||
```typescript
|
||||
class UserNotFoundError extends Error {
|
||||
readonly name = "UserNotFoundError"
|
||||
constructor(readonly userId: UserId) {
|
||||
super(`user ${userId} not found`)
|
||||
}
|
||||
}
|
||||
|
||||
class PermissionDeniedError extends Error {
|
||||
readonly name = "PermissionDeniedError"
|
||||
constructor(
|
||||
readonly userId: UserId,
|
||||
readonly requiredRole: string,
|
||||
) {
|
||||
super(`user ${userId} needs role ${requiredRole}`)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```typescript
|
||||
// BAD
|
||||
throw new Error("user not found")
|
||||
throw new Error("permission denied")
|
||||
|
||||
// GOOD
|
||||
throw new UserNotFoundError(userId)
|
||||
throw new PermissionDeniedError(userId, "admin")
|
||||
```
|
||||
|
||||
Always set `readonly name` explicitly — `instanceof` checks survive minification, but `error.name` is more reliable for logging and serialization.
|
||||
|
||||
---
|
||||
|
||||
## Result pattern — expected failures without exceptions
|
||||
|
||||
For failures that are **expected** (not found, validation), return a discriminated union instead of throwing.
|
||||
|
||||
```typescript
|
||||
type Result<T, E = Error> =
|
||||
| { readonly ok: true; readonly value: T }
|
||||
| { readonly ok: false; readonly error: E }
|
||||
|
||||
function ok<T>(value: T): Result<T, never> {
|
||||
return { ok: true, value }
|
||||
}
|
||||
|
||||
function err<E>(error: E): Result<never, E> {
|
||||
return { ok: false, error }
|
||||
}
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
```typescript
|
||||
type UserError =
|
||||
| { readonly kind: "not_found"; readonly id: UserId }
|
||||
| { readonly kind: "forbidden"; readonly reason: string }
|
||||
|
||||
function getUser(id: UserId): Result<User, UserError> {
|
||||
const user = db.find(id)
|
||||
if (!user) return err({ kind: "not_found", id })
|
||||
if (!user.active) return err({ kind: "forbidden", reason: "deactivated" })
|
||||
return ok(user)
|
||||
}
|
||||
|
||||
// Caller must handle both cases
|
||||
const result = getUser(userId)
|
||||
if (!result.ok) {
|
||||
switch (result.error.kind) {
|
||||
case "not_found":
|
||||
log.warn(`missing: ${result.error.id}`)
|
||||
break
|
||||
case "forbidden":
|
||||
log.error(`denied: ${result.error.reason}`)
|
||||
break
|
||||
default:
|
||||
assertNever(result.error)
|
||||
}
|
||||
return
|
||||
}
|
||||
const user = result.value // narrowed to User
|
||||
```
|
||||
|
||||
### When to use which
|
||||
|
||||
**The heuristic**: caller is 1-2 levels away and MUST handle it → Result. Error should propagate up many layers → throw.
|
||||
|
||||
| Scenario | Pattern | Why |
|
||||
|---|---|---|
|
||||
| Repository → service (caller handles it) | Result | Caller is right there, must handle both |
|
||||
| Validation at boundary (parsing input) | throw (Zod throws) | Propagates up to HTTP handler |
|
||||
| Infrastructure failure (network, OOM) | throw | Can't handle locally |
|
||||
| Service → service (deep internal) | throw (typed Error subclass) | Result boilerplate across many layers is worse |
|
||||
| HTTP handler → response | Catch errors, convert to response | Boundary code catches and translates |
|
||||
|
||||
**Practical tradeoff**: Result is safest (compiler forces handling) but creates boilerplate when every caller in a chain must check `.ok`. If the error would just propagate through 3+ layers unchanged, use a typed Error subclass instead.
|
||||
|
||||
### Library or roll your own?
|
||||
|
||||
Roll your own with the `Result`, `ok`, `err` above. It's 10 lines. Libraries like `neverthrow` add chaining (`.map`, `.andThen`) — use them only if you actually chain results frequently.
|
||||
|
||||
---
|
||||
|
||||
## Error cause — chain context
|
||||
|
||||
Use the `cause` option to chain errors without losing the original stack.
|
||||
|
||||
```typescript
|
||||
try {
|
||||
await db.query(sql)
|
||||
} catch (error) {
|
||||
throw new DatabaseError("query failed", { cause: error })
|
||||
}
|
||||
```
|
||||
|
||||
The `cause` is available on `error.cause` and shows up in stack traces.
|
||||
|
||||
---
|
||||
|
||||
## Exhaustive error handling at boundaries
|
||||
|
||||
HTTP handlers catch and translate:
|
||||
|
||||
```typescript
|
||||
app.onError((error, c) => {
|
||||
if (error instanceof UserNotFoundError) {
|
||||
return c.json({ error: error.message }, 404)
|
||||
}
|
||||
if (error instanceof PermissionDeniedError) {
|
||||
return c.json({ error: error.message }, 403)
|
||||
}
|
||||
console.error("unhandled:", error)
|
||||
return c.json({ error: "internal server error" }, 500)
|
||||
})
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Async error patterns
|
||||
|
||||
```typescript
|
||||
// Promise.allSettled — when partial failure is OK
|
||||
const results = await Promise.allSettled(urls.map(fetch))
|
||||
const successes = results
|
||||
.filter((r): r is PromiseFulfilledResult<Response> => r.status === "fulfilled")
|
||||
.map((r) => r.value)
|
||||
|
||||
// AbortSignal — cancellation
|
||||
async function fetchWithTimeout(url: string, ms: number): Promise<Response> {
|
||||
return fetch(url, { signal: AbortSignal.timeout(ms) })
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- MDN: [Error cause](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error/cause)
|
||||
- MDN: [Promise.allSettled](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/allSettled)
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
# Strict tsconfig + Biome
|
||||
|
||||
The canonical ultra-strict config. Copy-paste, then add your own paths.
|
||||
|
||||
---
|
||||
|
||||
## tsconfig.json
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"compilerOptions": {
|
||||
// ── Strict core ──────────────────────────────────────────
|
||||
"strict": true, // enables all strict* flags below
|
||||
// strict includes: strictNullChecks, strictFunctionTypes,
|
||||
// strictBindCallApply, strictPropertyInitialization,
|
||||
// noImplicitAny, noImplicitThis, alwaysStrict, useUnknownInCatchVariables
|
||||
|
||||
// ── Additional strict flags (NOT included in "strict") ──
|
||||
"noUncheckedIndexedAccess": true, // obj[key] is T | undefined, not T
|
||||
"exactOptionalPropertyTypes": true, // { x?: string } !== { x: string | undefined }
|
||||
"noFallthroughCasesInSwitch": true, // switch fall-through is an error
|
||||
"noPropertyAccessFromIndexSignature": true, // forces bracket notation for index sigs
|
||||
"forceConsistentCasingInFileNames": true, // prevents case-sensitivity bugs on macOS/Win
|
||||
|
||||
// ── Module system ────────────────────────────────────────
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"verbatimModuleSyntax": true, // forces `import type` for type-only imports
|
||||
"isolatedModules": true, // safe for esbuild / swc / Bun transpilation
|
||||
"esModuleInterop": true,
|
||||
"resolveJsonModule": true,
|
||||
|
||||
// ── Target ───────────────────────────────────────────────
|
||||
"target": "ESNext",
|
||||
"lib": ["ESNext"],
|
||||
|
||||
// ── Emit ─────────────────────────────────────────────────
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
|
||||
// ── Performance ──────────────────────────────────────────
|
||||
"skipLibCheck": true, // skip checking .d.ts files for speed
|
||||
"incremental": true
|
||||
},
|
||||
"include": ["src"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
```
|
||||
|
||||
### What each extra flag catches
|
||||
|
||||
| Flag | What it prevents |
|
||||
|---|---|
|
||||
| `noUncheckedIndexedAccess` | `arr[0]` is `T \| undefined`, not `T`. Forces you to check before using. |
|
||||
| `exactOptionalPropertyTypes` | `{ x?: string }` means "missing or string", NOT "string \| undefined". Assigns `undefined` explicitly? Type error. |
|
||||
| `noFallthroughCasesInSwitch` | Forgetting `break` / `return` in a switch case. |
|
||||
| `noPropertyAccessFromIndexSignature` | `obj.foo` on `Record<string, X>` is an error. Use `obj["foo"]`. |
|
||||
| `verbatimModuleSyntax` | Forces `import type { X }` for type-only imports. Prevents runtime import of types. |
|
||||
|
||||
### Bun-specific additions
|
||||
|
||||
For Bun projects, add to `compilerOptions`:
|
||||
```jsonc
|
||||
{
|
||||
"types": ["bun-types"],
|
||||
"moduleDetection": "force"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## biome.jsonc
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"$schema": "https://biomejs.dev/schemas/2.0.6/schema.json",
|
||||
"organizeImports": {
|
||||
"enabled": true
|
||||
},
|
||||
"linter": {
|
||||
"enabled": true,
|
||||
"rules": {
|
||||
"recommended": true,
|
||||
"suspicious": {
|
||||
"noExplicitAny": "error",
|
||||
"noConfusingVoidType": "error",
|
||||
"noFallthroughSwitchClause": "error"
|
||||
},
|
||||
"style": {
|
||||
"noDefaultExport": "error",
|
||||
"useImportType": "error",
|
||||
"noNonNullAssertion": "error",
|
||||
"useEnumInitializers": "off",
|
||||
"noParameterAssign": "error"
|
||||
},
|
||||
"correctness": {
|
||||
"noUnusedVariables": "error",
|
||||
"noUnusedImports": "error"
|
||||
},
|
||||
"complexity": {
|
||||
"noBannedTypes": "error"
|
||||
}
|
||||
}
|
||||
},
|
||||
"formatter": {
|
||||
"enabled": true,
|
||||
"indentStyle": "space",
|
||||
"indentWidth": 2,
|
||||
"lineWidth": 100
|
||||
},
|
||||
"javascript": {
|
||||
"formatter": {
|
||||
"quoteStyle": "double",
|
||||
"semicolons": "asNeeded"
|
||||
}
|
||||
},
|
||||
"files": {
|
||||
"ignore": ["node_modules", "dist", "build", ".next", ".nuxt", "coverage"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Key Biome rules
|
||||
|
||||
| Rule | What |
|
||||
|---|---|
|
||||
| `noExplicitAny` | `any` in annotations is an error |
|
||||
| `noNonNullAssertion` | `x!` is an error |
|
||||
| `noDefaultExport` | Forces named exports |
|
||||
| `useImportType` | Forces `import type` for type-only imports |
|
||||
| `noParameterAssign` | No mutation of function parameters |
|
||||
|
||||
---
|
||||
|
||||
## CI gate
|
||||
|
||||
```bash
|
||||
bunx biome check .
|
||||
bunx tsc --noEmit
|
||||
bun test
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- TypeScript: [tsconfig reference](https://www.typescriptlang.org/tsconfig)
|
||||
- Biome: [configuration](https://biomejs.dev/reference/configuration/)
|
||||
- Total TypeScript: [tsconfig cheat sheet](https://www.totaltypescript.com/tsconfig-cheat-sheet)
|
||||
@@ -0,0 +1,196 @@
|
||||
# Type Patterns
|
||||
|
||||
How to use TypeScript's type system to catch bugs at compile time.
|
||||
|
||||
---
|
||||
|
||||
## Branded types — distinct primitives
|
||||
|
||||
Same runtime type, different meaning. The compiler prevents mixing.
|
||||
|
||||
```typescript
|
||||
declare const brand: unique symbol
|
||||
type Brand<T, B extends string> = T & { readonly [brand]: B }
|
||||
|
||||
type UserId = Brand<string, "UserId">
|
||||
type OrderId = Brand<string, "OrderId">
|
||||
type Milliseconds = Brand<number, "Milliseconds">
|
||||
type Seconds = Brand<number, "Seconds">
|
||||
|
||||
function UserId(value: string): UserId { return value as UserId }
|
||||
function OrderId(value: string): OrderId { return value as OrderId }
|
||||
|
||||
function getUser(id: UserId): User { ... }
|
||||
|
||||
getUser(UserId("abc")) // OK
|
||||
getUser(OrderId("abc")) // type error: OrderId is not UserId
|
||||
getUser("abc") // type error: string is not UserId
|
||||
```
|
||||
|
||||
With Zod (preferred at boundaries):
|
||||
```typescript
|
||||
import { z } from "zod"
|
||||
|
||||
const UserIdSchema = z.string().uuid().brand("UserId")
|
||||
type UserId = z.infer<typeof UserIdSchema>
|
||||
```
|
||||
|
||||
**Use when**: IDs, indices, units of measurement — any pair where swapping is a bug.
|
||||
|
||||
---
|
||||
|
||||
## as const — literal types from values
|
||||
|
||||
Freezes a value to its narrowest possible type. The foundation for enum-free TypeScript.
|
||||
|
||||
```typescript
|
||||
const ROLES = ["admin", "user", "guest"] as const
|
||||
type Role = (typeof ROLES)[number] // "admin" | "user" | "guest"
|
||||
|
||||
const HTTP_STATUS = {
|
||||
OK: 200,
|
||||
NOT_FOUND: 404,
|
||||
INTERNAL: 500,
|
||||
} as const
|
||||
type HttpStatus = (typeof HTTP_STATUS)[keyof typeof HTTP_STATUS] // 200 | 404 | 500
|
||||
```
|
||||
|
||||
**Use when**: fixed set of constants. Replaces `enum` entirely.
|
||||
**Skip when**: the set is open-ended or user-defined.
|
||||
|
||||
---
|
||||
|
||||
## satisfies — validate without widening
|
||||
|
||||
Type-checks a value against a type while preserving the literal type. Best of both worlds.
|
||||
|
||||
```typescript
|
||||
type Config = Record<string, string | number>
|
||||
|
||||
// BAD — widens to Record<string, string | number>
|
||||
const config: Config = { api: "https://api.example.com", timeout: 30 }
|
||||
config.api // string | number — lost the narrowing
|
||||
|
||||
// GOOD — validates AND preserves literal types
|
||||
const config = {
|
||||
api: "https://api.example.com",
|
||||
timeout: 30,
|
||||
} satisfies Config
|
||||
config.api // string (narrowed)
|
||||
config.timeout // number (narrowed)
|
||||
```
|
||||
|
||||
**Use when**: you want type validation on a value without losing narrowing.
|
||||
|
||||
---
|
||||
|
||||
## Discriminated unions — algebraic data types
|
||||
|
||||
Model every outcome as a type. Force the caller to handle all cases.
|
||||
|
||||
```typescript
|
||||
type GetUserResult =
|
||||
| { readonly kind: "found"; readonly user: User }
|
||||
| { readonly kind: "not_found"; readonly id: UserId }
|
||||
| { readonly kind: "forbidden"; readonly reason: string }
|
||||
```
|
||||
|
||||
The `kind` field (or `type`, `status`, `_tag`) is the discriminant. TypeScript narrows on it automatically.
|
||||
|
||||
---
|
||||
|
||||
## Exhaustive switch — assertNever
|
||||
|
||||
Every switch on a discriminated union ends with a default that calls `assertNever`.
|
||||
|
||||
```typescript
|
||||
function assertNever(x: never): never {
|
||||
throw new Error(`Unexpected value: ${JSON.stringify(x)}`)
|
||||
}
|
||||
|
||||
function handleResult(result: GetUserResult): string {
|
||||
switch (result.kind) {
|
||||
case "found":
|
||||
return result.user.name
|
||||
case "not_found":
|
||||
return `No user ${result.id}`
|
||||
case "forbidden":
|
||||
return `Denied: ${result.reason}`
|
||||
default:
|
||||
return assertNever(result)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Add a new variant to `GetUserResult`? The compiler errors on the `assertNever` call until you handle it.
|
||||
|
||||
---
|
||||
|
||||
## Narrowing — let the compiler follow your logic
|
||||
|
||||
TypeScript narrows types through `typeof`, `instanceof`, `in`, equality checks, and discriminants.
|
||||
|
||||
```typescript
|
||||
function process(value: string | number | null): string {
|
||||
if (value === null) return "nothing"
|
||||
// compiler knows: string | number
|
||||
|
||||
if (typeof value === "string") return value.toUpperCase()
|
||||
// compiler knows: number
|
||||
|
||||
return String(value * 2)
|
||||
}
|
||||
```
|
||||
|
||||
### Custom type guards
|
||||
|
||||
```typescript
|
||||
function isNonNull<T>(value: T | null | undefined): value is T {
|
||||
return value != null
|
||||
}
|
||||
|
||||
const items = [1, null, 2, undefined, 3]
|
||||
const clean = items.filter(isNonNull) // number[]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## import type — separate values from types
|
||||
|
||||
Always use `import type` for type-only imports. Enforced by `verbatimModuleSyntax`.
|
||||
|
||||
```typescript
|
||||
import type { User, Config } from "./types" // erased at runtime
|
||||
import { createUser } from "./services" // kept at runtime
|
||||
```
|
||||
|
||||
For mixed imports:
|
||||
```typescript
|
||||
import { createUser, type User } from "./users"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Utility types — quick reference
|
||||
|
||||
| Need | Use |
|
||||
|---|---|
|
||||
| All properties readonly | `Readonly<T>` |
|
||||
| All properties optional | `Partial<T>` |
|
||||
| All properties required | `Required<T>` |
|
||||
| Pick specific properties | `Pick<T, "a" \| "b">` |
|
||||
| Omit specific properties | `Omit<T, "a" \| "b">` |
|
||||
| Key-value map | `Record<K, V>` |
|
||||
| Extract from union | `Extract<T, U>` |
|
||||
| Exclude from union | `Exclude<T, U>` |
|
||||
| Return type of function | `ReturnType<typeof fn>` |
|
||||
| Parameters of function | `Parameters<typeof fn>` |
|
||||
| Awaited type | `Awaited<Promise<T>>` → `T` |
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- TypeScript Handbook: [Narrowing](https://www.typescriptlang.org/docs/handbook/2/narrowing.html)
|
||||
- TypeScript Handbook: [Template Literal Types](https://www.typescriptlang.org/docs/handbook/2/template-literal-types.html)
|
||||
- Total TypeScript: [as const](https://www.totaltypescript.com/as-const)
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env bash
|
||||
# No-excuse rule checker for Go files.
|
||||
# Mirrors the philosophy of python-programmer / typescript-programmer / rust-programmer scripts:
|
||||
# only rules that can be enforced via pure text matching live here.
|
||||
# Everything semantic is on golangci-lint + nilaway + go test -race.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "Usage: $0 <file.go> [file.go ...]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
violations=0
|
||||
report() {
|
||||
local file="$1"
|
||||
local line="$2"
|
||||
local rule="$3"
|
||||
local detail="$4"
|
||||
echo "::error file=${file},line=${line}::[${rule}] ${detail}" >&2
|
||||
violations=$((violations + 1))
|
||||
}
|
||||
|
||||
is_test_file() {
|
||||
case "$1" in
|
||||
*_test.go) return 0 ;;
|
||||
esac
|
||||
return 1
|
||||
}
|
||||
|
||||
is_generated_file() {
|
||||
local file="$1"
|
||||
case "$file" in
|
||||
*.pb.go|*.connect.go|*.gen.go) return 0 ;;
|
||||
*_string.go) return 0 ;;
|
||||
esac
|
||||
# First-line check for "Code generated ... DO NOT EDIT." (the official marker)
|
||||
if [ -f "$file" ]; then
|
||||
head -n 5 "$file" 2>/dev/null | grep -qE "^// Code generated .* DO NOT EDIT\.$" && return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
for file in "$@"; do
|
||||
[ -f "$file" ] || continue
|
||||
case "$file" in
|
||||
*.go) ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
|
||||
if is_generated_file "$file"; then
|
||||
continue
|
||||
fi
|
||||
|
||||
in_test=0
|
||||
if is_test_file "$file"; then
|
||||
in_test=1
|
||||
fi
|
||||
|
||||
line_no=0
|
||||
while IFS= read -r raw_line || [ -n "$raw_line" ]; do
|
||||
line_no=$((line_no + 1))
|
||||
line="$raw_line"
|
||||
|
||||
# Strip line comments before pattern checks
|
||||
# (block comments are not handled — keep the rules robust to that limitation).
|
||||
code_only="${line%%//*}"
|
||||
|
||||
# ── Exemption marker: // no-excuse-ok: <reason> ──────────────────
|
||||
if [[ "$line" =~ //[[:space:]]*no-excuse-ok:[[:space:]]*.+ ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── Rule: no `_ = err` (silent error swallow) ────────────────────
|
||||
# The errcheck linter catches most of these but the `_ = err` form
|
||||
# specifically slips through if used with named returns.
|
||||
if [[ "$code_only" =~ ^[[:space:]]*_[[:space:]]*=[[:space:]]*err[[:space:]]*$ ]] ||
|
||||
[[ "$code_only" =~ ^[[:space:]]*_[[:space:]]*=[[:space:]]*err[[:space:]]*[^a-zA-Z0-9_].*$ ]]; then
|
||||
if [ "$in_test" -eq 0 ]; then
|
||||
report "$file" "$line_no" "silent-err" "discarding err with '_ = err' — handle the error"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Rule: no `panic(` in non-test, non-main code ─────────────────
|
||||
# Allowed in main(), allowed in tests, allowed with explicit marker.
|
||||
if [[ "$code_only" =~ [^a-zA-Z0-9_]panic\( ]] || [[ "$code_only" =~ ^[[:space:]]*panic\( ]]; then
|
||||
if [ "$in_test" -eq 0 ]; then
|
||||
# main package main.go is the one exception
|
||||
pkg_line=$(head -n 5 "$file" 2>/dev/null | grep -m1 "^package ")
|
||||
if [[ "$pkg_line" != "package main" ]]; then
|
||||
report "$file" "$line_no" "panic-in-lib" "panic outside main/test — return error instead"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Rule: no `log.Fatal` / `log.Panic` in library code ───────────
|
||||
if [[ "$code_only" =~ log\.(Fatal|Panic)(f|ln)?\( ]]; then
|
||||
if [ "$in_test" -eq 0 ]; then
|
||||
pkg_line=$(head -n 5 "$file" 2>/dev/null | grep -m1 "^package ")
|
||||
if [[ "$pkg_line" != "package main" ]]; then
|
||||
report "$file" "$line_no" "log-fatal-in-lib" "log.Fatal/Panic outside main — return error"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Rule: no init() functions ─────────────────────────────────
|
||||
# init() ruins testability and creates hidden global state.
|
||||
# Exception: //go:build constraint files and generated code.
|
||||
if [[ "$code_only" =~ ^func[[:space:]]+init\(\)[[:space:]]*\{ ]]; then
|
||||
report "$file" "$line_no" "no-init-func" "init() ruins testability — use explicit constructor"
|
||||
fi
|
||||
|
||||
# ── Rule: no `time.Sleep` in non-test code ──────────────────────
|
||||
if [[ "$code_only" =~ time\.Sleep\( ]]; then
|
||||
if [ "$in_test" -eq 0 ]; then
|
||||
report "$file" "$line_no" "time-sleep" "time.Sleep in production code — use ticker/timer with ctx"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Rule: no `context.Background()` inside functions (only in main/init/test) ──
|
||||
if [[ "$code_only" =~ context\.Background\(\) ]]; then
|
||||
if [ "$in_test" -eq 0 ]; then
|
||||
pkg_line=$(head -n 5 "$file" 2>/dev/null | grep -m1 "^package ")
|
||||
if [[ "$pkg_line" != "package main" ]]; then
|
||||
report "$file" "$line_no" "ctx-background-in-lib" "context.Background() outside main — propagate ctx as parameter"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Rule: no `interface{}` (use `any`, the alias from Go 1.18+) ──
|
||||
if [[ "$code_only" =~ interface\{\} ]]; then
|
||||
report "$file" "$line_no" "old-interface-empty" "use 'any' instead of 'interface{}' (Go 1.18+)"
|
||||
fi
|
||||
|
||||
# ── Rule: no bare `fmt.Println` for logging (use slog) ───────────
|
||||
# Acceptable in main.go (CLI output) and tests. Reject in libraries.
|
||||
if [[ "$code_only" =~ fmt\.(Print|Println|Printf)\( ]]; then
|
||||
if [ "$in_test" -eq 0 ]; then
|
||||
pkg_line=$(head -n 5 "$file" 2>/dev/null | grep -m1 "^package ")
|
||||
if [[ "$pkg_line" != "package main" ]]; then
|
||||
report "$file" "$line_no" "fmt-print-in-lib" "fmt.Print* in library — use slog for structured logs"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Rule: no `nolint` directive without reason ───────────────────
|
||||
if [[ "$line" =~ //nolint(:|$| ) ]]; then
|
||||
if ! [[ "$line" =~ //nolint:[a-zA-Z0-9_,-]+[[:space:]]+//[[:space:]]*[^[:space:]] ]]; then
|
||||
report "$file" "$line_no" "nolint-no-reason" "//nolint requires a // reason after the linter list"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Rule: no TODO / FIXME without an issue link or owner ─────────
|
||||
# Check the full line — TODOs live in comments, which $code_only has stripped.
|
||||
if echo "$line" | grep -qE '(TODO|FIXME|XXX)([[:space:]]|:)'; then
|
||||
if ! echo "$line" | grep -qE '(TODO|FIXME|XXX).*[(@[]'; then
|
||||
report "$file" "$line_no" "todo-no-owner" "TODO/FIXME requires (#issue) or @owner attribution"
|
||||
fi
|
||||
fi
|
||||
done < "$file"
|
||||
done
|
||||
|
||||
if [ "$violations" -gt 0 ]; then
|
||||
echo "" >&2
|
||||
echo "go-programmer: $violations violation(s). Run also:" >&2
|
||||
echo " gofumpt -l ." >&2
|
||||
echo " golangci-lint run --timeout 5m ./..." >&2
|
||||
echo " nilaway ./..." >&2
|
||||
echo " go test -race -shuffle=on -count=1 ./..." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "go-programmer: no-excuse rules passed for $# file(s)."
|
||||
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# dependencies = [
|
||||
# "typer",
|
||||
# "rich",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
# ─── How to run ───
|
||||
# 1. Install uv (if not installed):
|
||||
# curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
# 2. Run:
|
||||
# uv run new-project.py myservice
|
||||
# uv run new-project.py myservice --module github.com/your-org/myservice
|
||||
# ──────────────────
|
||||
#
|
||||
# Creates a new Go project with the canonical strict layout:
|
||||
# - go.mod with go 1.23
|
||||
# - .golangci.yml (v2, strict bundle)
|
||||
# - Taskfile.yml (fmt + lint + test + build)
|
||||
# - cmd/server/main.go entrypoint
|
||||
# - internal/{cmd,config,api,domain,obs} skeletons
|
||||
# - .github/workflows/ci.yml
|
||||
#
|
||||
# Templates live in ./templates/ — keep this script under 250 pure LOC.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from string import Template
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
|
||||
console = Console(stderr=True)
|
||||
|
||||
TEMPLATES_DIR = Path(__file__).parent / "templates"
|
||||
|
||||
|
||||
def _render(template_file: str, **subs: str) -> str:
|
||||
"""Read a template file and apply $placeholder substitutions.
|
||||
|
||||
Uses string.Template ($name) so Go/YAML curly braces stay literal.
|
||||
"""
|
||||
raw = (TEMPLATES_DIR / template_file).read_text()
|
||||
if not subs:
|
||||
return raw
|
||||
return Template(raw).substitute(**subs)
|
||||
|
||||
|
||||
# (template-file → relative output path; is_format = .format() is run)
|
||||
FILES: list[tuple[str, str, bool]] = [
|
||||
(".golangci.yml", ".golangci.yml", False),
|
||||
("Taskfile.yml", "Taskfile.yml", False),
|
||||
(".editorconfig", ".editorconfig", False),
|
||||
("gitignore", ".gitignore", False),
|
||||
("ci.yml", ".github/workflows/ci.yml", False),
|
||||
("run.go", "internal/cmd/run.go", False),
|
||||
("config.go", "internal/config/config.go", False),
|
||||
("main.go.tmpl", "cmd/server/main.go", True),
|
||||
("AGENTS.md.tmpl", "AGENTS.md", True),
|
||||
("README.md.tmpl", "README.md", True),
|
||||
]
|
||||
|
||||
|
||||
def _init_go_module(project_dir: Path, module: str) -> None:
|
||||
try:
|
||||
subprocess.run(
|
||||
["go", "mod", "init", module],
|
||||
cwd=project_dir,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
console.print(f" [dim]ran[/] go mod init {module}")
|
||||
except (subprocess.CalledProcessError, FileNotFoundError) as e:
|
||||
console.print(f" [yellow]warn[/] go mod init failed ({e}); writing fallback go.mod")
|
||||
(project_dir / "go.mod").write_text(f"module {module}\n\ngo 1.23\n")
|
||||
|
||||
|
||||
def _create_layout(project_dir: Path) -> None:
|
||||
"""Create the canonical internal/ tree."""
|
||||
subdirs = [
|
||||
"cmd/server",
|
||||
"internal/cmd",
|
||||
"internal/config",
|
||||
"internal/api",
|
||||
"internal/domain",
|
||||
"internal/obs",
|
||||
".github/workflows",
|
||||
]
|
||||
for sd in subdirs:
|
||||
(project_dir / sd).mkdir(parents=True)
|
||||
|
||||
|
||||
def _write_files(project_dir: Path, name: str, module: str, purpose: str) -> None:
|
||||
"""Render every template into the project tree."""
|
||||
for tmpl_name, out_rel, is_format in FILES:
|
||||
subs = (
|
||||
{"name": name, "module": module, "short_purpose": purpose}
|
||||
if is_format
|
||||
else {}
|
||||
)
|
||||
content = _render(tmpl_name, **subs)
|
||||
out_path = project_dir / out_rel
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text(content)
|
||||
console.print(f" [dim]wrote[/] {out_rel}")
|
||||
|
||||
|
||||
def main(
|
||||
name: str,
|
||||
path: str = typer.Option(".", help="Parent dir"),
|
||||
module: str = typer.Option("", help="Go module path; default: <name>"),
|
||||
purpose: str = typer.Option("HTTP", help="Short purpose for AGENTS.md"),
|
||||
) -> None:
|
||||
"""Scaffold a new Go project with the strict toolchain."""
|
||||
project_dir = Path(path) / name
|
||||
if project_dir.exists():
|
||||
console.print(f"[red]✗[/red] {project_dir} already exists")
|
||||
sys.exit(1)
|
||||
|
||||
module_path = module or name
|
||||
|
||||
project_dir.mkdir(parents=True)
|
||||
_create_layout(project_dir)
|
||||
_init_go_module(project_dir, module_path)
|
||||
_write_files(project_dir, name, module_path, purpose)
|
||||
|
||||
console.print(f"\n[bold green]Done![/] cd {project_dir}")
|
||||
console.print(" go get github.com/caarlos0/env/v11")
|
||||
console.print(" task # fmt + lint + test")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
typer.run(main)
|
||||
@@ -0,0 +1,13 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
indent_style = tab
|
||||
indent_size = 4
|
||||
end_of_line = lf
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
|
||||
[*.{yml,yaml,json,md}]
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
@@ -0,0 +1,95 @@
|
||||
version: "2"
|
||||
|
||||
run:
|
||||
timeout: 5m
|
||||
tests: true
|
||||
modules-download-mode: readonly
|
||||
|
||||
linters:
|
||||
default: none
|
||||
enable:
|
||||
- govet
|
||||
- staticcheck
|
||||
- errcheck
|
||||
- errorlint
|
||||
- nilerr
|
||||
- nilnil
|
||||
- bodyclose
|
||||
- rowserrcheck
|
||||
- sqlclosecheck
|
||||
- contextcheck
|
||||
- fatcontext
|
||||
- copyloopvar
|
||||
- intrange
|
||||
- usetesting
|
||||
- testifylint
|
||||
- gofumpt
|
||||
- goimports
|
||||
- whitespace
|
||||
- misspell
|
||||
- unconvert
|
||||
- unparam
|
||||
- ineffassign
|
||||
- dupword
|
||||
- gocognit
|
||||
- gocyclo
|
||||
- funlen
|
||||
- lll
|
||||
- nestif
|
||||
- dupl
|
||||
- revive
|
||||
- unused
|
||||
- exhaustive
|
||||
- gosec
|
||||
- sloglint
|
||||
- perfsprint
|
||||
- prealloc
|
||||
- makezero
|
||||
|
||||
linters-settings:
|
||||
errcheck:
|
||||
check-type-assertions: true
|
||||
check-blank: true
|
||||
errorlint:
|
||||
errorf: true
|
||||
asserts: true
|
||||
comparison: true
|
||||
gocognit:
|
||||
min-complexity: 25
|
||||
gocyclo:
|
||||
min-complexity: 15
|
||||
funlen:
|
||||
lines: 90
|
||||
statements: 60
|
||||
lll:
|
||||
line-length: 120
|
||||
tab-width: 4
|
||||
nestif:
|
||||
min-complexity: 4
|
||||
exhaustive:
|
||||
default-signifies-exhaustive: false
|
||||
check: [switch, map]
|
||||
sloglint:
|
||||
no-mixed-args: true
|
||||
attr-only: true
|
||||
no-global: all
|
||||
context: scope
|
||||
static-msg: true
|
||||
no-raw-keys: true
|
||||
key-naming-case: snake
|
||||
|
||||
formatters:
|
||||
enable:
|
||||
- gofumpt
|
||||
- goimports
|
||||
|
||||
issues:
|
||||
max-issues-per-linter: 0
|
||||
max-same-issues: 0
|
||||
exclude-rules:
|
||||
- path: _test\\.go
|
||||
linters: [funlen, lll, dupl, gosec]
|
||||
- path: \\.pb\\.go$
|
||||
linters: [all]
|
||||
- path: \\.connect\\.go$
|
||||
linters: [all]
|
||||
@@ -0,0 +1,24 @@
|
||||
# AGENTS.md
|
||||
|
||||
Go 1.23+ $short_purpose service.
|
||||
|
||||
## Commands
|
||||
- `task` — fmt + lint + test
|
||||
- `task build` — produce ./bin/server
|
||||
- `task ci` — full CI pipeline locally
|
||||
|
||||
## Architecture
|
||||
- `cmd/server/main.go` — entrypoint, ≤50 LOC
|
||||
- `internal/cmd/` — root command, signal wiring
|
||||
- `internal/api/` — HTTP handlers + middleware (gin)
|
||||
- `internal/domain/` — smart-constructor types, no I/O
|
||||
- `internal/store/` — DB layer (sqlc-generated, never hand-edited)
|
||||
- `internal/config/` — env-driven Config
|
||||
- `internal/obs/` — slog setup, observability
|
||||
|
||||
## Conventions
|
||||
- `slog` for all logs; never `log.*`, never `fmt.Println` in libs
|
||||
- `context.Context` first arg for every public function with I/O
|
||||
- Errors wrapped with `%w`; check with `errors.Is/As`
|
||||
- 250 pure LOC ceiling per file
|
||||
- Tests follow Given/When/Then; less mock the better
|
||||
@@ -0,0 +1,12 @@
|
||||
# $name
|
||||
|
||||
Bootstrapped with the `programming` skill's Go scaffold.
|
||||
|
||||
## Run
|
||||
|
||||
```bash
|
||||
task # fmt + lint + test
|
||||
task run # build + run server
|
||||
```
|
||||
|
||||
See `AGENTS.md` for architecture conventions.
|
||||
@@ -0,0 +1,40 @@
|
||||
version: '3'
|
||||
|
||||
vars:
|
||||
BINARY: server
|
||||
PKG: ./cmd/server
|
||||
|
||||
tasks:
|
||||
default:
|
||||
deps: [fmt, lint, test]
|
||||
|
||||
fmt:
|
||||
cmds:
|
||||
- gofumpt -w .
|
||||
- goimports -w -local "$(go list -m)" .
|
||||
|
||||
lint:
|
||||
cmds:
|
||||
- golangci-lint run --timeout 5m ./...
|
||||
- nilaway ./... || true
|
||||
|
||||
test:
|
||||
cmds:
|
||||
- go test -race -shuffle=on -count=1 ./...
|
||||
|
||||
test-cover:
|
||||
cmds:
|
||||
- go test -race -shuffle=on -count=1 -coverprofile=coverage.out ./...
|
||||
- go tool cover -html=coverage.out -o coverage.html
|
||||
|
||||
build:
|
||||
cmds:
|
||||
- go build -trimpath -ldflags="-s -w" -o bin/{{.BINARY}} {{.PKG}}
|
||||
|
||||
run:
|
||||
deps: [build]
|
||||
cmds:
|
||||
- ./bin/{{.BINARY}}
|
||||
|
||||
ci:
|
||||
deps: [fmt, lint, test, build]
|
||||
@@ -0,0 +1,37 @@
|
||||
name: ci
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
ci:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.23'
|
||||
cache: true
|
||||
|
||||
- name: Install tools
|
||||
run: |
|
||||
go install mvdan.cc/gofumpt@latest
|
||||
go install github.com/golangci/golangci-lint/cmd/golangci-lint@v2.0.0
|
||||
go install go.uber.org/nilaway/cmd/nilaway@latest
|
||||
go install github.com/go-task/task/v3/cmd/task@latest
|
||||
|
||||
- name: Format check
|
||||
run: gofumpt -l . | (! grep .)
|
||||
|
||||
- name: Lint
|
||||
run: golangci-lint run --timeout 5m ./...
|
||||
|
||||
- name: Nilaway
|
||||
run: nilaway ./... || true
|
||||
|
||||
- name: Test
|
||||
run: go test -race -shuffle=on -count=1 ./...
|
||||
|
||||
- name: Build
|
||||
run: go build -trimpath ./...
|
||||
@@ -0,0 +1,24 @@
|
||||
// Package config loads typed config from env.
|
||||
package config
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/caarlos0/env/v11"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Host string `env:"HOST" envDefault:"0.0.0.0"`
|
||||
Port int `env:"PORT" envDefault:"8080"`
|
||||
ShutdownTimeout time.Duration `env:"SHUTDOWN_TIMEOUT" envDefault:"20s"`
|
||||
LogLevel string `env:"LOG_LEVEL" envDefault:"info"`
|
||||
LogFormat string `env:"LOG_FORMAT" envDefault:"json"`
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
var cfg Config
|
||||
if err := env.Parse(&cfg); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
bin/
|
||||
coverage.out
|
||||
coverage.html
|
||||
*.test
|
||||
*.prof
|
||||
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
|
||||
.env
|
||||
.env.local
|
||||
|
||||
*.pem
|
||||
*.key
|
||||
@@ -0,0 +1,22 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"$module/internal/cmd"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(),
|
||||
syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
if err := cmd.Execute(ctx); err != nil {
|
||||
slog.Error("fatal", slog.Any("err", err))
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
// Package cmd wires the root command and subcommands.
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Execute runs the root command. Wire cobra/subcommands here.
|
||||
func Execute(ctx context.Context) error {
|
||||
slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||
slog.InfoContext(ctx, "starting")
|
||||
return nil
|
||||
}
|
||||
+687
@@ -0,0 +1,687 @@
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# dependencies = []
|
||||
# ///
|
||||
# noqa: SIZE_OK — single self-contained checker, splitting adds import ceremony for no readability gain
|
||||
"""Check Python files for no-excuse violations.
|
||||
|
||||
The python-programmer skill enforces these rules. Run after editing.
|
||||
|
||||
Rules:
|
||||
cast-any - cast(Any, ...) / cast(typing.Any, ...) / typing.cast(Any, ...)
|
||||
type-ignore - `# type: ignore` comments (any variant)
|
||||
pyright-ignore - `# pyright: ignore` comments (any variant)
|
||||
bare-except - `except:` with no class
|
||||
silent-except - `except X: pass` or `except X: ...` (single statement)
|
||||
no-asyncio - `import asyncio` / `from asyncio import ...`
|
||||
Opt out per import line: trailing `# noqa: ANYIO_OK`
|
||||
no-pandas - `import pandas` / `from pandas import ...`
|
||||
Opt out per import line: trailing `# noqa: PANDAS_OK`
|
||||
mutable-dataclass - @dataclass without frozen=True
|
||||
Opt out: trailing `# noqa: MUTABLE_OK`
|
||||
missing-slots - @dataclass without slots=True
|
||||
Opt out: trailing `# noqa: SLOTS_OK`
|
||||
raw-dict-return - function returns bare `dict` type
|
||||
Opt out: trailing `# noqa: DICT_OK`
|
||||
missing-assert-never - match statement without assert_never in default case
|
||||
Opt out: `# noqa: MATCH_OK` on the match line
|
||||
generic-exception - raise ValueError/TypeError/RuntimeError with bare string
|
||||
Opt out: trailing `# noqa: GENERIC_ERR_OK`
|
||||
no-object - `object` used as type annotation (param, return, variable)
|
||||
Opt out: trailing `# noqa: OBJECT_OK`
|
||||
if-elif-on-variant - isinstance/enum-comparison if/elif chain (should be match/case)
|
||||
Opt out: trailing `# noqa: IF_VARIANT_OK`
|
||||
oversized-module - file exceeds 250 pure LOC (non-blank, non-comment)
|
||||
Opt out: `# noqa: SIZE_OK` in first 10 lines
|
||||
broad-except - `except Exception` / `except BaseException` (too broad)
|
||||
Opt out: trailing `# noqa: BROAD_EXCEPT_OK`
|
||||
|
||||
Usage:
|
||||
check-no-excuse-rules.py <file-or-dir>...
|
||||
|
||||
Exit codes:
|
||||
0 - no violations
|
||||
1 - one or more violations
|
||||
2 - input error (path missing, etc.)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import io
|
||||
import re
|
||||
import sys
|
||||
import tokenize
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
EXCLUDED_DIRS = frozenset({
|
||||
".git", ".hg", ".svn", ".venv", "venv", "env", ".env",
|
||||
"__pycache__", ".tox", ".nox", "dist", "build", ".eggs",
|
||||
".ruff_cache", ".mypy_cache", ".pytest_cache", ".basedpyright",
|
||||
"node_modules",
|
||||
})
|
||||
|
||||
SUPPRESSION_RE = re.compile(r"#\s*(type|pyright)\s*:\s*ignore\b")
|
||||
ANYIO_OK_RE = re.compile(r"#\s*noqa:\s*ANYIO_OK\b")
|
||||
PANDAS_OK_RE = re.compile(r"#\s*noqa:\s*PANDAS_OK\b")
|
||||
|
||||
BANNED_IMPORTS: dict[str, tuple[str, re.Pattern[str], str]] = {
|
||||
"asyncio": (
|
||||
"no-asyncio",
|
||||
ANYIO_OK_RE,
|
||||
"import asyncio - use anyio (opt out: trailing `# noqa: ANYIO_OK`)",
|
||||
),
|
||||
"pandas": (
|
||||
"no-pandas",
|
||||
PANDAS_OK_RE,
|
||||
"import pandas - use polars (opt out: trailing `# noqa: PANDAS_OK`)",
|
||||
),
|
||||
}
|
||||
|
||||
# Opt-out patterns for new Rust-like rules
|
||||
MUTABLE_OK_RE: re.Pattern[str] = re.compile(r"#\s*noqa:\s*MUTABLE_OK")
|
||||
SLOTS_OK_RE: re.Pattern[str] = re.compile(r"#\s*noqa:\s*SLOTS_OK")
|
||||
DICT_OK_RE: re.Pattern[str] = re.compile(r"#\s*noqa:\s*DICT_OK")
|
||||
MATCH_OK_RE: re.Pattern[str] = re.compile(r"#\s*noqa:\s*MATCH_OK")
|
||||
GENERIC_ERR_OK_RE: re.Pattern[str] = re.compile(r"#\s*noqa:\s*GENERIC_ERR_OK")
|
||||
OBJECT_OK_RE: re.Pattern[str] = re.compile(r"#\s*noqa:\s*OBJECT_OK")
|
||||
IF_VARIANT_OK_RE: re.Pattern[str] = re.compile(r"#\s*noqa:\s*IF_VARIANT_OK")
|
||||
SIZE_OK_RE: re.Pattern[str] = re.compile(r"#\s*noqa:\s*SIZE_OK")
|
||||
BROAD_EXCEPT_OK_RE: re.Pattern[str] = re.compile(r"#\s*noqa:\s*BROAD_EXCEPT_OK")
|
||||
|
||||
PURE_LOC_LIMIT: int = 250
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Violation:
|
||||
rule: str
|
||||
file: Path
|
||||
line: int
|
||||
col: int
|
||||
message: str
|
||||
|
||||
def render(self) -> str:
|
||||
return f"{self.file}:{self.line}:{self.col}: [{self.rule}] {self.message}"
|
||||
|
||||
|
||||
def discover_files(inputs: Iterable[Path]) -> list[Path]:
|
||||
seen: set[Path] = set()
|
||||
for raw in inputs:
|
||||
path = raw.resolve()
|
||||
if not path.exists():
|
||||
print(f"check-no-excuse-rules: input does not exist: {path}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
if path.is_file():
|
||||
if path.suffix == ".py":
|
||||
seen.add(path)
|
||||
continue
|
||||
for child in path.rglob("*.py"):
|
||||
if any(part in EXCLUDED_DIRS for part in child.parts):
|
||||
continue
|
||||
seen.add(child)
|
||||
return sorted(seen)
|
||||
|
||||
|
||||
def is_any_node(node: ast.AST) -> bool:
|
||||
if isinstance(node, ast.Name):
|
||||
return node.id == "Any"
|
||||
if isinstance(node, ast.Attribute):
|
||||
return node.attr == "Any"
|
||||
return False
|
||||
|
||||
|
||||
def is_cast_callable(node: ast.AST) -> bool:
|
||||
if isinstance(node, ast.Name):
|
||||
return node.id == "cast"
|
||||
if isinstance(node, ast.Attribute):
|
||||
return node.attr == "cast"
|
||||
return False
|
||||
|
||||
|
||||
def find_node_violations(tree: ast.AST, file: Path) -> list[Violation]:
|
||||
violations: list[Violation] = []
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if (
|
||||
isinstance(node, ast.Call)
|
||||
and is_cast_callable(node.func)
|
||||
and node.args
|
||||
and is_any_node(node.args[0])
|
||||
):
|
||||
violations.append(Violation(
|
||||
rule="cast-any",
|
||||
file=file,
|
||||
line=node.lineno,
|
||||
col=node.col_offset + 1,
|
||||
message="cast(Any, ...) - narrow with isinstance/TypeGuard or use a Protocol/TypedDict",
|
||||
))
|
||||
|
||||
if isinstance(node, ast.ExceptHandler):
|
||||
if node.type is None:
|
||||
violations.append(Violation(
|
||||
rule="bare-except",
|
||||
file=file,
|
||||
line=node.lineno,
|
||||
col=node.col_offset + 1,
|
||||
message="bare `except:` - catch the narrowest exception you mean",
|
||||
))
|
||||
|
||||
if len(node.body) != 1:
|
||||
continue
|
||||
|
||||
body = node.body[0]
|
||||
if isinstance(body, ast.Pass):
|
||||
violations.append(Violation(
|
||||
rule="silent-except",
|
||||
file=file,
|
||||
line=body.lineno,
|
||||
col=body.col_offset + 1,
|
||||
message="silent `except: pass` - log, re-raise, or actually handle the error",
|
||||
))
|
||||
elif (
|
||||
isinstance(body, ast.Expr)
|
||||
and isinstance(body.value, ast.Constant)
|
||||
and body.value.value is Ellipsis
|
||||
):
|
||||
violations.append(Violation(
|
||||
rule="silent-except",
|
||||
file=file,
|
||||
line=body.lineno,
|
||||
col=body.col_offset + 1,
|
||||
message="silent `except: ...` - log, re-raise, or actually handle the error",
|
||||
))
|
||||
|
||||
return violations
|
||||
|
||||
|
||||
def find_import_violations(tree: ast.AST, source_lines: list[str], file: Path) -> list[Violation]:
|
||||
violations: list[Violation] = []
|
||||
|
||||
def line_text(lineno: int) -> str:
|
||||
index = lineno - 1
|
||||
return source_lines[index] if 0 <= index < len(source_lines) else ""
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import): # noqa: IF_VARIANT_OK — filtering walk, not closed union
|
||||
for alias in node.names:
|
||||
top = alias.name.split(".")[0]
|
||||
if top not in BANNED_IMPORTS:
|
||||
continue
|
||||
rule, opt_re, message = BANNED_IMPORTS[top]
|
||||
if opt_re.search(line_text(node.lineno)):
|
||||
continue
|
||||
violations.append(Violation(
|
||||
rule=rule,
|
||||
file=file,
|
||||
line=node.lineno,
|
||||
col=node.col_offset + 1,
|
||||
message=message,
|
||||
))
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
top = (node.module or "").split(".")[0]
|
||||
if top not in BANNED_IMPORTS:
|
||||
continue
|
||||
rule, opt_re, message = BANNED_IMPORTS[top]
|
||||
if opt_re.search(line_text(node.lineno)):
|
||||
continue
|
||||
violations.append(Violation(
|
||||
rule=rule,
|
||||
file=file,
|
||||
line=node.lineno,
|
||||
col=node.col_offset + 1,
|
||||
message=message,
|
||||
))
|
||||
|
||||
return violations
|
||||
|
||||
|
||||
def find_comment_violations(source: str, file: Path) -> list[Violation]:
|
||||
"""Use tokenize so we don't false-match `# type: ignore` inside string literals."""
|
||||
violations: list[Violation] = []
|
||||
try:
|
||||
tokens = list(tokenize.generate_tokens(io.StringIO(source).readline))
|
||||
except tokenize.TokenError as exc:
|
||||
print(f"check-no-excuse-rules: tokenize failed for {file}: {exc}", file=sys.stderr)
|
||||
return violations
|
||||
|
||||
for tok in tokens:
|
||||
if tok.type != tokenize.COMMENT:
|
||||
continue
|
||||
match = SUPPRESSION_RE.search(tok.string)
|
||||
if not match:
|
||||
continue
|
||||
kind = match.group(1)
|
||||
rule = "type-ignore" if kind == "type" else "pyright-ignore"
|
||||
violations.append(Violation(
|
||||
rule=rule,
|
||||
file=file,
|
||||
line=tok.start[0],
|
||||
col=tok.start[1] + match.start() + 1,
|
||||
message=f"`# {kind}: ignore` - fix the underlying type instead",
|
||||
))
|
||||
return violations
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# Rust-like pattern checks
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _has_keyword(decorator_node: ast.Call, keyword: str) -> bool | None:
|
||||
"""Check if a decorator call has a specific keyword argument.
|
||||
|
||||
Returns True if keyword is True, False if keyword is False or absent, None if not a Call.
|
||||
"""
|
||||
for kw in decorator_node.keywords:
|
||||
if kw.arg == keyword and isinstance(kw.value, ast.Constant):
|
||||
return bool(kw.value.value)
|
||||
return False
|
||||
|
||||
|
||||
def _is_dataclass_decorator(node: ast.expr) -> tuple[bool, ast.Call | None]:
|
||||
"""Return (is_dataclass, call_node_or_None)."""
|
||||
if isinstance(node, ast.Name) and node.id == "dataclass":
|
||||
return True, None
|
||||
if isinstance(node, ast.Attribute) and node.attr == "dataclass":
|
||||
return True, None
|
||||
if isinstance(node, ast.Call):
|
||||
inner, _ = _is_dataclass_decorator(node.func)
|
||||
if inner:
|
||||
return True, node
|
||||
return False, None
|
||||
|
||||
|
||||
def find_dataclass_violations(
|
||||
tree: ast.Module, source_lines: list[str], file: Path,
|
||||
) -> list[Violation]:
|
||||
"""Check @dataclass decorators for frozen=True and slots=True."""
|
||||
violations: list[Violation] = []
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.ClassDef):
|
||||
continue
|
||||
for dec in node.decorator_list:
|
||||
is_dc, call_node = _is_dataclass_decorator(dec)
|
||||
if not is_dc:
|
||||
continue
|
||||
|
||||
# Get the line of the decorator for opt-out check
|
||||
dec_line = source_lines[dec.lineno - 1] if dec.lineno <= len(source_lines) else ""
|
||||
|
||||
if call_node is not None:
|
||||
has_frozen = _has_keyword(call_node, "frozen")
|
||||
has_slots = _has_keyword(call_node, "slots")
|
||||
else:
|
||||
# bare @dataclass with no arguments
|
||||
has_frozen = False
|
||||
has_slots = False
|
||||
|
||||
if not has_frozen and not MUTABLE_OK_RE.search(dec_line):
|
||||
violations.append(Violation(
|
||||
rule="mutable-dataclass",
|
||||
file=file,
|
||||
line=dec.lineno,
|
||||
col=dec.col_offset + 1,
|
||||
message=f"class {node.name}: @dataclass without frozen=True",
|
||||
))
|
||||
|
||||
if not has_slots and not SLOTS_OK_RE.search(dec_line):
|
||||
violations.append(Violation(
|
||||
rule="missing-slots",
|
||||
file=file,
|
||||
line=dec.lineno,
|
||||
col=dec.col_offset + 1,
|
||||
message=f"class {node.name}: @dataclass without slots=True",
|
||||
))
|
||||
return violations
|
||||
|
||||
|
||||
def find_dict_return_violations(
|
||||
tree: ast.Module, source_lines: list[str], file: Path,
|
||||
) -> list[Violation]:
|
||||
"""Check for functions returning bare `dict` type."""
|
||||
violations: list[Violation] = []
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
continue
|
||||
ret = node.returns
|
||||
if ret is None:
|
||||
continue
|
||||
# Check for bare `dict` return annotation
|
||||
is_bare_dict = (
|
||||
(isinstance(ret, ast.Name) and ret.id == "dict")
|
||||
or (isinstance(ret, ast.Attribute) and ret.attr == "dict")
|
||||
)
|
||||
if not is_bare_dict:
|
||||
continue
|
||||
|
||||
func_line = source_lines[node.lineno - 1] if node.lineno <= len(source_lines) else ""
|
||||
if DICT_OK_RE.search(func_line):
|
||||
continue
|
||||
|
||||
violations.append(Violation(
|
||||
rule="raw-dict-return",
|
||||
file=file,
|
||||
line=node.lineno,
|
||||
col=node.col_offset + 1,
|
||||
message=f"`{node.name}` returns bare dict - use TypedDict/dataclass/Pydantic model",
|
||||
))
|
||||
return violations
|
||||
|
||||
|
||||
def find_match_violations(
|
||||
tree: ast.Module, source_lines: list[str], file: Path,
|
||||
) -> list[Violation]:
|
||||
"""Check match statements for assert_never in default case."""
|
||||
violations: list[Violation] = []
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Match):
|
||||
continue
|
||||
|
||||
match_line = source_lines[node.lineno - 1] if node.lineno <= len(source_lines) else ""
|
||||
if MATCH_OK_RE.search(match_line):
|
||||
continue
|
||||
|
||||
has_assert_never = False
|
||||
for case in node.cases:
|
||||
# Wildcard: `case _:` -> MatchAs(pattern=None, name=None)
|
||||
# `case _ as x:` -> MatchAs(pattern=MatchAs(pattern=None, name=None), name="x")
|
||||
pattern = case.pattern
|
||||
is_wildcard = (
|
||||
isinstance(pattern, ast.MatchAs)
|
||||
and (
|
||||
pattern.pattern is None
|
||||
or (
|
||||
isinstance(pattern.pattern, ast.MatchAs)
|
||||
and pattern.pattern.pattern is None
|
||||
and pattern.pattern.name is None
|
||||
)
|
||||
)
|
||||
)
|
||||
if not is_wildcard:
|
||||
continue
|
||||
# Check if body contains assert_never call
|
||||
for stmt in case.body:
|
||||
if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Call):
|
||||
func = stmt.value.func
|
||||
if (
|
||||
(isinstance(func, ast.Name) and func.id == "assert_never")
|
||||
or (isinstance(func, ast.Attribute) and func.attr == "assert_never")
|
||||
):
|
||||
has_assert_never = True
|
||||
break
|
||||
|
||||
if not has_assert_never:
|
||||
violations.append(Violation(
|
||||
rule="missing-assert-never",
|
||||
file=file,
|
||||
line=node.lineno,
|
||||
col=node.col_offset + 1,
|
||||
message="match without `case _: assert_never(x)` default",
|
||||
))
|
||||
return violations
|
||||
|
||||
|
||||
def find_generic_exception_violations(
|
||||
tree: ast.Module, source_lines: list[str], file: Path,
|
||||
) -> list[Violation]:
|
||||
"""Check for raise ValueError/TypeError/RuntimeError with bare string or f-string."""
|
||||
GENERIC_EXCEPTIONS = {"ValueError", "TypeError", "RuntimeError", "KeyError"}
|
||||
violations: list[Violation] = []
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Raise) or node.exc is None:
|
||||
continue
|
||||
exc = node.exc
|
||||
# Match: raise SomeError("string literal")
|
||||
if not isinstance(exc, ast.Call):
|
||||
continue
|
||||
func = exc.func
|
||||
exc_name: str | None = None
|
||||
if isinstance(func, ast.Name) and func.id in GENERIC_EXCEPTIONS:
|
||||
exc_name = func.id
|
||||
elif isinstance(func, ast.Attribute) and func.attr in GENERIC_EXCEPTIONS:
|
||||
exc_name = func.attr
|
||||
if exc_name is None:
|
||||
continue
|
||||
# Check if all arguments are string literals or f-strings
|
||||
if not exc.args:
|
||||
continue
|
||||
all_str = all(
|
||||
(isinstance(arg, ast.Constant) and isinstance(arg.value, str))
|
||||
or isinstance(arg, ast.JoinedStr)
|
||||
for arg in exc.args
|
||||
)
|
||||
if not all_str:
|
||||
continue
|
||||
|
||||
raise_line = source_lines[node.lineno - 1] if node.lineno <= len(source_lines) else ""
|
||||
if GENERIC_ERR_OK_RE.search(raise_line):
|
||||
continue
|
||||
|
||||
violations.append(Violation(
|
||||
rule="generic-exception",
|
||||
file=file,
|
||||
line=node.lineno,
|
||||
col=node.col_offset + 1,
|
||||
message=f"`raise {exc_name}(\"...\")` - define a typed error class instead",
|
||||
))
|
||||
return violations
|
||||
|
||||
|
||||
def _is_isinstance_test(node: ast.expr) -> bool:
|
||||
"""Check if node is an isinstance() call."""
|
||||
return (
|
||||
isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Name)
|
||||
and node.func.id == "isinstance"
|
||||
)
|
||||
|
||||
|
||||
def _is_enum_comparison(node: ast.expr) -> bool:
|
||||
"""Check if node is `x == Enum.VALUE` or `x is Enum.VALUE`."""
|
||||
if isinstance(node, ast.Compare) and len(node.ops) == 1:
|
||||
op = node.ops[0]
|
||||
if isinstance(op, (ast.Eq, ast.Is)):
|
||||
comparator = node.comparators[0]
|
||||
# x == Enum.VALUE (attribute access on the right)
|
||||
if isinstance(comparator, ast.Attribute):
|
||||
return True
|
||||
# Enum.VALUE == x (attribute access on the left)
|
||||
if isinstance(node.left, ast.Attribute):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def find_object_annotation_violations(
|
||||
tree: ast.Module, source_lines: list[str], file: Path,
|
||||
) -> list[Violation]:
|
||||
"""Check for `object` used as a type annotation."""
|
||||
violations: list[Violation] = []
|
||||
|
||||
def _check_annotation(ann: ast.expr | None) -> None:
|
||||
if ann is None:
|
||||
return
|
||||
for child in ast.walk(ann):
|
||||
if isinstance(child, ast.Name) and child.id == "object":
|
||||
line = source_lines[child.lineno - 1] if child.lineno <= len(source_lines) else ""
|
||||
if OBJECT_OK_RE.search(line):
|
||||
return
|
||||
violations.append(Violation(
|
||||
rule="no-object",
|
||||
file=file,
|
||||
line=child.lineno,
|
||||
col=child.col_offset + 1,
|
||||
message="`object` as type annotation \u2014 use Protocol, TypeVar, or union",
|
||||
))
|
||||
|
||||
for node in ast.walk(tree):
|
||||
match node: # noqa: MATCH_OK — filtering walk, not discriminating a closed union
|
||||
case ast.FunctionDef() | ast.AsyncFunctionDef():
|
||||
all_args = (
|
||||
node.args.args
|
||||
+ node.args.posonlyargs
|
||||
+ node.args.kwonlyargs
|
||||
)
|
||||
for arg in all_args:
|
||||
_check_annotation(arg.annotation)
|
||||
if node.args.vararg:
|
||||
_check_annotation(node.args.vararg.annotation)
|
||||
if node.args.kwarg:
|
||||
_check_annotation(node.args.kwarg.annotation)
|
||||
_check_annotation(node.returns)
|
||||
case ast.AnnAssign():
|
||||
_check_annotation(node.annotation)
|
||||
return violations
|
||||
|
||||
|
||||
def find_if_elif_variant_violations(
|
||||
tree: ast.Module, source_lines: list[str], file: Path,
|
||||
) -> list[Violation]:
|
||||
"""Check for if/elif chains on isinstance or enum comparison."""
|
||||
violations: list[Violation] = []
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.If):
|
||||
continue
|
||||
|
||||
line = source_lines[node.lineno - 1] if node.lineno <= len(source_lines) else ""
|
||||
if IF_VARIANT_OK_RE.search(line):
|
||||
continue
|
||||
|
||||
is_variant_test = _is_isinstance_test(node.test) or _is_enum_comparison(node.test)
|
||||
if not is_variant_test:
|
||||
continue
|
||||
|
||||
# Must have at least one elif that is also a variant test
|
||||
orelse = node.orelse
|
||||
while orelse and len(orelse) == 1 and isinstance(orelse[0], ast.If):
|
||||
elif_node = orelse[0]
|
||||
if _is_isinstance_test(elif_node.test) or _is_enum_comparison(elif_node.test):
|
||||
violations.append(Violation(
|
||||
rule="if-elif-on-variant",
|
||||
file=file,
|
||||
line=node.lineno,
|
||||
col=node.col_offset + 1,
|
||||
message="isinstance/enum if/elif chain \u2014 use match/case + assert_never",
|
||||
))
|
||||
break
|
||||
orelse = elif_node.orelse
|
||||
return violations
|
||||
|
||||
|
||||
def find_broad_except_violations(
|
||||
tree: ast.Module, source_lines: list[str], file: Path,
|
||||
) -> list[Violation]:
|
||||
"""Check for except Exception / except BaseException (too broad)."""
|
||||
BROAD_EXCEPTIONS = {"Exception", "BaseException"}
|
||||
violations: list[Violation] = []
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.ExceptHandler):
|
||||
continue
|
||||
if node.type is None:
|
||||
continue # already caught by bare-except
|
||||
|
||||
exc_name: str | None = None
|
||||
if isinstance(node.type, ast.Name) and node.type.id in BROAD_EXCEPTIONS:
|
||||
exc_name = node.type.id
|
||||
elif isinstance(node.type, ast.Attribute) and node.type.attr in BROAD_EXCEPTIONS:
|
||||
exc_name = node.type.attr
|
||||
if exc_name is None:
|
||||
continue
|
||||
|
||||
line = source_lines[node.lineno - 1] if node.lineno <= len(source_lines) else ""
|
||||
if BROAD_EXCEPT_OK_RE.search(line):
|
||||
continue
|
||||
|
||||
violations.append(Violation(
|
||||
rule="broad-except",
|
||||
file=file,
|
||||
line=node.lineno,
|
||||
col=node.col_offset + 1,
|
||||
message=f"`except {exc_name}` is too broad \u2014 catch the specific exception you expect",
|
||||
))
|
||||
return violations
|
||||
|
||||
|
||||
def find_oversized_module_violations(
|
||||
source_lines: list[str], file: Path,
|
||||
) -> list[Violation]:
|
||||
"""Check if file exceeds 250 pure LOC (non-blank, non-comment)."""
|
||||
# File-level opt-out in first 10 lines (shebang + script metadata can push it down)
|
||||
for line in source_lines[:10]:
|
||||
if SIZE_OK_RE.search(line):
|
||||
return []
|
||||
|
||||
pure_loc = sum(
|
||||
1 for line in source_lines
|
||||
if line.strip() and not line.strip().startswith("#")
|
||||
)
|
||||
if pure_loc > PURE_LOC_LIMIT:
|
||||
return [Violation(
|
||||
rule="oversized-module",
|
||||
file=file,
|
||||
line=1,
|
||||
col=1,
|
||||
message=f"{pure_loc} pure LOC (limit: {PURE_LOC_LIMIT}) \u2014 split by responsibility",
|
||||
)]
|
||||
return []
|
||||
|
||||
|
||||
def check_file(file: Path) -> list[Violation]:
|
||||
source = file.read_text(encoding="utf-8")
|
||||
try:
|
||||
tree = ast.parse(source, filename=str(file))
|
||||
except SyntaxError as exc:
|
||||
return [Violation(
|
||||
rule="syntax-error",
|
||||
file=file,
|
||||
line=exc.lineno or 1,
|
||||
col=exc.offset or 1,
|
||||
message=f"SyntaxError: {exc.msg}",
|
||||
)]
|
||||
|
||||
source_lines = source.splitlines()
|
||||
return [
|
||||
*find_node_violations(tree, file),
|
||||
*find_import_violations(tree, source_lines, file),
|
||||
*find_comment_violations(source, file),
|
||||
*find_dataclass_violations(tree, source_lines, file),
|
||||
*find_dict_return_violations(tree, source_lines, file),
|
||||
*find_match_violations(tree, source_lines, file),
|
||||
*find_generic_exception_violations(tree, source_lines, file),
|
||||
*find_object_annotation_violations(tree, source_lines, file),
|
||||
*find_if_elif_variant_violations(tree, source_lines, file),
|
||||
*find_oversized_module_violations(source_lines, file),
|
||||
*find_broad_except_violations(tree, source_lines, file),
|
||||
]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) < 2:
|
||||
print("usage: check-no-excuse-rules.py <file-or-dir>...", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
files = discover_files(Path(arg) for arg in sys.argv[1:])
|
||||
if not files:
|
||||
print("check-no-excuse-rules: no .py files found", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
violations: list[Violation] = []
|
||||
for file in files:
|
||||
violations.extend(check_file(file))
|
||||
|
||||
if not violations:
|
||||
print(f"no violations in {len(files)} file(s)")
|
||||
return 0
|
||||
|
||||
for violation in violations:
|
||||
print(violation.render(), file=sys.stderr)
|
||||
print(
|
||||
f"\n{len(violations)} violation(s) in {len(files)} file(s)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# dependencies = [
|
||||
# "typer",
|
||||
# "rich",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
# ─── How to run ───
|
||||
# 1. Install uv (if not installed):
|
||||
# curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
# 2. Run:
|
||||
# uv run new-project.py myproject
|
||||
# uv run new-project.py myproject --path ./workspace
|
||||
# uv run new-project.py myproject --lib # library (publishable)
|
||||
# ──────────────────
|
||||
|
||||
"""Scaffold a new Python project with ultra-strict config from pyproject-strict.md.
|
||||
|
||||
Creates via `uv init`, then injects basedpyright + ruff ALL + pytest config.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import typer
|
||||
from rich import print as rprint
|
||||
|
||||
# ── Strict tool config (from pyproject-strict.md) ──
|
||||
|
||||
TOOL_CONFIG = '''
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"basedpyright>=1.21",
|
||||
"ruff>=0.8",
|
||||
"pytest>=8",
|
||||
"pytest-cov>=5",
|
||||
]
|
||||
|
||||
[tool.basedpyright]
|
||||
typeCheckingMode = "all"
|
||||
pythonVersion = "3.13"
|
||||
reportMissingTypeStubs = false
|
||||
reportUnknownMemberType = false
|
||||
reportUnknownArgumentType = false
|
||||
reportUnknownVariableType = false
|
||||
reportUnknownLambdaType = false
|
||||
reportUnknownParameterType = false
|
||||
reportMissingParameterType = false
|
||||
reportUnnecessaryIsInstance = false
|
||||
reportUnusedCallResult = false
|
||||
reportImplicitOverride = false
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py313"
|
||||
line-length = 120
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["ALL"]
|
||||
ignore = [
|
||||
"COM812", # trailing comma (conflicts with formatter)
|
||||
"ISC001", # single-line string concat (conflicts with formatter)
|
||||
"D1", # undocumented-public-* (too noisy early on)
|
||||
"ANN101", # deprecated: self annotation
|
||||
"ANN102", # deprecated: cls annotation
|
||||
"S101", # assert used (pytest needs it)
|
||||
"PLR2004", # magic-value-comparison (test data)
|
||||
"FBT", # boolean-trap (too strict for CLIs)
|
||||
"TD", # flake8-todos (noisy)
|
||||
"FIX", # fixme (noisy)
|
||||
]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"tests/**/*.py" = ["S101", "PLR2004", "SLF001", "D", "ARG", "ANN"]
|
||||
|
||||
[tool.ruff.lint.pydocstyle]
|
||||
convention = "google"
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
addopts = "-ra --strict-markers --strict-config"
|
||||
'''
|
||||
|
||||
GITIGNORE = """\
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.so
|
||||
.venv/
|
||||
dist/
|
||||
*.egg-info/
|
||||
.coverage
|
||||
htmlcov/
|
||||
.basedpyright/
|
||||
.ruff_cache/
|
||||
"""
|
||||
|
||||
|
||||
def main(
|
||||
name: str = typer.Argument(help="Project name"),
|
||||
path: Path = typer.Option(Path("."), "--path", "-p", help="Parent directory"),
|
||||
lib: bool = typer.Option(False, "--lib", help="Create as publishable library (uv init --lib)"),
|
||||
) -> None:
|
||||
"""Create a new Python project with ultra-strict config."""
|
||||
project_dir = path / name
|
||||
|
||||
if project_dir.exists():
|
||||
rprint(f"[red]Error:[/red] {project_dir} already exists")
|
||||
raise SystemExit(1)
|
||||
|
||||
# Run uv init
|
||||
cmd = ["uv", "init", "--lib" if lib else "--app", str(project_dir)]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
rprint(f"[red]uv init failed:[/red] {result.stderr}")
|
||||
raise SystemExit(1)
|
||||
|
||||
# Read existing pyproject.toml
|
||||
pyproject_path = project_dir / "pyproject.toml"
|
||||
content = pyproject_path.read_text()
|
||||
|
||||
# Remove the default [dependency-groups] if uv init created one
|
||||
# (we'll replace it with our strict version)
|
||||
lines = content.splitlines(keepends=True)
|
||||
filtered: list[str] = []
|
||||
skip = False
|
||||
for line in lines:
|
||||
if line.strip().startswith("[dependency-groups]"):
|
||||
skip = True
|
||||
continue
|
||||
if skip and line.strip().startswith("["):
|
||||
skip = False
|
||||
if not skip:
|
||||
filtered.append(line)
|
||||
|
||||
content = "".join(filtered).rstrip("\n") + "\n"
|
||||
|
||||
# Append strict tool config
|
||||
content += TOOL_CONFIG
|
||||
|
||||
pyproject_path.write_text(content)
|
||||
|
||||
# Add dev dependencies
|
||||
subprocess.run(
|
||||
["uv", "add", "--dev", "basedpyright", "ruff", "pytest", "pytest-cov"],
|
||||
cwd=project_dir,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
# Create tests directory
|
||||
tests_dir = project_dir / "tests"
|
||||
tests_dir.mkdir(exist_ok=True)
|
||||
(tests_dir / "__init__.py").touch()
|
||||
|
||||
# Overwrite .gitignore
|
||||
(project_dir / ".gitignore").write_text(GITIGNORE)
|
||||
|
||||
# Create py.typed marker for libraries
|
||||
if lib:
|
||||
src_dir = project_dir / "src" / name.replace("-", "_")
|
||||
if src_dir.exists():
|
||||
(src_dir / "py.typed").touch()
|
||||
|
||||
rprint(f"[green]✓[/green] Created: [bold]{project_dir}[/bold]")
|
||||
rprint(f" cd {name} && uv sync && uv run basedpyright . && uv run ruff check .")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
typer.run(main)
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# dependencies = [
|
||||
# "typer",
|
||||
# "rich",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
# ─── How to run ───
|
||||
# 1. Install uv (if not installed):
|
||||
# curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
# 2. Run:
|
||||
# uv run new-script.py my_tool
|
||||
# uv run new-script.py my_tool --output ./scripts/my_tool.py
|
||||
# uv run new-script.py my_tool --deps 'httpx2[http2,brotli,zstd]' --deps rich --deps polars
|
||||
# uv run new-script.py my_tool --py 3.13
|
||||
# ──────────────────
|
||||
|
||||
"""Generate a PEP 723 Python script with all boilerplate pre-filled.
|
||||
|
||||
Creates a new .py file with:
|
||||
- uv shebang
|
||||
- PEP 723 inline metadata (requires-python + dependencies)
|
||||
- Mandatory "How to run" comment block
|
||||
- from __future__ import annotations
|
||||
- main() + if __name__ guard
|
||||
|
||||
By default writes to a temp directory and prints the path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import typer
|
||||
from rich import print as rprint
|
||||
|
||||
|
||||
TEMPLATE = '''\
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# requires-python = ">={python_version}"
|
||||
# dependencies = [
|
||||
{deps_block}# ]
|
||||
# ///
|
||||
|
||||
# ─── How to run ───
|
||||
# 1. Install uv (if not installed):
|
||||
# curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
# 2. Run directly (no venv, no pip install needed):
|
||||
# uv run {filename} {args_hint}
|
||||
# 3. Or make executable and run:
|
||||
# chmod +x {filename} && ./{filename}
|
||||
# ──────────────────
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""TODO: implement."""
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
'''
|
||||
|
||||
|
||||
def main(
|
||||
name: str = typer.Argument(help="Script name (without .py extension)"),
|
||||
output: Path | None = typer.Option(None, "--output", "-o", help="Output path. Default: OS temp directory."),
|
||||
deps: list[str] = typer.Option([], "--deps", "-d", help="Dependencies to include (repeat --deps for each)."),
|
||||
py: str = typer.Option("3.13", "--py", help="Minimum Python version."),
|
||||
) -> None:
|
||||
"""Generate a new PEP 723 script with all boilerplate pre-filled."""
|
||||
filename = f"{name}.py" if not name.endswith(".py") else name
|
||||
stem = filename.removesuffix(".py")
|
||||
|
||||
if output is not None:
|
||||
dest = Path(output)
|
||||
else:
|
||||
tmp_dir = Path(tempfile.gettempdir()) / "uv-scripts"
|
||||
tmp_dir.mkdir(exist_ok=True)
|
||||
dest = tmp_dir / filename
|
||||
|
||||
dep_list = deps or []
|
||||
if dep_list:
|
||||
deps_block = "".join(f'# "{d}",\n' for d in dep_list)
|
||||
else:
|
||||
deps_block = '# # add deps here, e.g.: "httpx2[http2,brotli,zstd]"\n'
|
||||
|
||||
content = TEMPLATE.format(
|
||||
python_version=py,
|
||||
deps_block=deps_block,
|
||||
filename=filename,
|
||||
args_hint="",
|
||||
)
|
||||
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_text(content)
|
||||
|
||||
# Make executable on Unix
|
||||
if sys.platform != "win32":
|
||||
st = dest.stat()
|
||||
dest.chmod(st.st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
|
||||
|
||||
rprint(f"[green]✓[/green] Created: [bold]{dest}[/bold]")
|
||||
rprint(f" Run: [cyan]uv run {dest}[/cyan]")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
typer.run(main)
|
||||
+296
@@ -0,0 +1,296 @@
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# dependencies = []
|
||||
# ///
|
||||
#
|
||||
# How to run:
|
||||
# uv run --script check-no-excuse-rules.py src/lib.rs src/main.rs
|
||||
# uv run --script check-no-excuse-rules.py src/ # recursively finds .rs files
|
||||
# uv run --script check-no-excuse-rules.py . # entire tree
|
||||
#
|
||||
# No-excuse rule checker for Rust files — Python rewrite of check-no-excuse-rules.sh.
|
||||
# Only rules enforceable via pure text matching live here.
|
||||
# Everything semantic is on clippy + miri + nextest.
|
||||
#
|
||||
# Rules:
|
||||
# unwrap .unwrap() outside tests without // SAFE-UNWRAP:
|
||||
# expect .expect() outside tests without // SAFE-EXPECT:
|
||||
# placeholder-macro todo!/unimplemented!/unreachable!/unreachable_unchecked! in committed code
|
||||
# box-dyn-error Box<dyn Error> in non-test code
|
||||
# lib-panic panic!() in library code
|
||||
# unsafe-no-safety unsafe { without // SAFETY: in preceding 5 lines
|
||||
# unjustified-clippy-allow #[allow(clippy::...)] without // CLIPPY-ALLOW:
|
||||
# narrowing-as-cast possible narrowing 'as' cast
|
||||
#
|
||||
# Opt-out: place the appropriate comment on the previous line:
|
||||
# // SAFE-UNWRAP: <reason>
|
||||
# // SAFE-EXPECT: <reason>
|
||||
# // SAFETY: <reason> (for unsafe blocks, within 5 lines above)
|
||||
# // CLIPPY-ALLOW: <reason>
|
||||
#
|
||||
# Test paths (exempt from unwrap/expect/placeholder/box-dyn-error/lib-panic):
|
||||
# tests/, benches/, examples/, build.rs, *_test.rs, #[cfg(test)] regions
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Patterns (compiled once)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
RE_UNWRAP = re.compile(r"\.unwrap\(\)")
|
||||
RE_EXPECT = re.compile(r"\.expect\(")
|
||||
RE_PLACEHOLDER = re.compile(r"\b(todo!|unimplemented!|unreachable!|unreachable_unchecked!)")
|
||||
RE_BOX_DYN_ERROR = re.compile(r"Box<dyn\s+Error")
|
||||
RE_PANIC = re.compile(r"\bpanic!\(")
|
||||
RE_UNSAFE_BLOCK = re.compile(r"\bunsafe\s*\{")
|
||||
RE_CLIPPY_ALLOW = re.compile(r"#\[allow\(clippy::")
|
||||
RE_CFG_TEST = re.compile(r"#\[cfg\(test\)\]")
|
||||
RE_SAFE_UNWRAP = re.compile(r"//\s*SAFE-UNWRAP:")
|
||||
RE_SAFE_EXPECT = re.compile(r"//\s*SAFE-EXPECT:")
|
||||
RE_SAFETY = re.compile(r"//\s*SAFETY:")
|
||||
RE_CLIPPY_ALLOW_JUST = re.compile(r"//\s*CLIPPY-ALLOW:")
|
||||
|
||||
# Narrowing cast: (wider) as (narrower)
|
||||
# Wider types that lose bits when cast to narrower targets.
|
||||
# No leading \b — must match e.g. `999u64 as u32` where a digit precedes the type.
|
||||
_WIDER = r"(?:u16|u32|u64|u128|usize|i16|i32|i64|i128|isize)"
|
||||
_NARROWER = r"(?:u8|u16|u32|i8|i16|i32)"
|
||||
RE_NARROWING_CAST = re.compile(
|
||||
rf"{_WIDER}\s+as\s+{_NARROWER}"
|
||||
)
|
||||
|
||||
# Test-path fragments.
|
||||
_TEST_PATH_PARTS = {"tests", "benches", "examples"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
violations = 0
|
||||
|
||||
|
||||
def report(file: str, line: int, rule: str, detail: str) -> None:
|
||||
"""Emit a GitHub-Actions-compatible error annotation to stderr."""
|
||||
global violations
|
||||
print(f"::error file={file},line={line}::[{rule}] {detail}", file=sys.stderr)
|
||||
violations += 1
|
||||
|
||||
|
||||
def is_test_path(path: Path) -> bool:
|
||||
"""Return True if *path* is in a test/bench/example directory or is a test file."""
|
||||
parts = path.parts
|
||||
for part in parts:
|
||||
if part in _TEST_PATH_PARTS:
|
||||
return True
|
||||
if path.name == "build.rs":
|
||||
return True
|
||||
if path.name.endswith("_test.rs"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_lib_path(file: Path) -> bool:
|
||||
"""Heuristic: is this file library code (not main.rs, not src/bin/*)."""
|
||||
parts = path_parts_str(file)
|
||||
# Must live under src/
|
||||
if "src" not in parts:
|
||||
return False
|
||||
if file.name == "main.rs":
|
||||
return False
|
||||
# src/bin/* is binary code
|
||||
try:
|
||||
src_idx = parts.index("src")
|
||||
if src_idx + 1 < len(parts) and parts[src_idx + 1] == "bin":
|
||||
return False
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def path_parts_str(p: Path) -> list[str]:
|
||||
return list(p.parts)
|
||||
|
||||
|
||||
def strip_line_comment(line: str) -> str:
|
||||
"""Return the portion of *line* before any ``//`` line comment.
|
||||
|
||||
This is a crude heuristic — it does not handle ``//`` inside string
|
||||
literals, but matches the behaviour of the bash version.
|
||||
"""
|
||||
idx = line.find("//")
|
||||
if idx == -1:
|
||||
return line
|
||||
return line[:idx]
|
||||
|
||||
|
||||
def collect_rs_files(args: list[str]) -> list[Path]:
|
||||
"""Expand CLI arguments: files are kept as-is, directories are walked."""
|
||||
result: list[Path] = []
|
||||
for arg in args:
|
||||
p = Path(arg)
|
||||
if p.is_file():
|
||||
if p.suffix == ".rs":
|
||||
result.append(p)
|
||||
elif p.is_dir():
|
||||
result.extend(sorted(p.rglob("*.rs")))
|
||||
# Ignore non-existent / non-.rs
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main checker
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_file(file: Path) -> None:
|
||||
in_test_file = is_test_path(file)
|
||||
in_cfg_test = False
|
||||
cfg_test_brace_depth = 0
|
||||
|
||||
try:
|
||||
lines = file.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
except OSError as exc:
|
||||
print(f"warning: cannot read {file}: {exc}", file=sys.stderr)
|
||||
return
|
||||
|
||||
for line_no_0, raw_line in enumerate(lines):
|
||||
line_no = line_no_0 + 1 # 1-indexed
|
||||
|
||||
# --- #[cfg(test)] region tracker ---
|
||||
if RE_CFG_TEST.search(raw_line):
|
||||
in_cfg_test = True
|
||||
cfg_test_brace_depth = 0
|
||||
|
||||
if in_cfg_test:
|
||||
opens = raw_line.count("{")
|
||||
closes = raw_line.count("}")
|
||||
cfg_test_brace_depth += opens - closes
|
||||
if cfg_test_brace_depth <= 0 and not RE_CFG_TEST.search(raw_line):
|
||||
in_cfg_test = False
|
||||
|
||||
exempt = in_test_file or in_cfg_test
|
||||
code_only = strip_line_comment(raw_line)
|
||||
|
||||
if not exempt:
|
||||
# .unwrap()
|
||||
if RE_UNWRAP.search(code_only):
|
||||
prev = lines[line_no_0 - 1] if line_no_0 > 0 else ""
|
||||
if not RE_SAFE_UNWRAP.search(prev):
|
||||
report(
|
||||
str(file), line_no, "unwrap",
|
||||
".unwrap() outside tests - use ? / ok_or / pattern match "
|
||||
"or annotate previous line with // SAFE-UNWRAP: <reason>",
|
||||
)
|
||||
|
||||
# .expect(...)
|
||||
if RE_EXPECT.search(code_only):
|
||||
prev = lines[line_no_0 - 1] if line_no_0 > 0 else ""
|
||||
if not RE_SAFE_EXPECT.search(prev):
|
||||
report(
|
||||
str(file), line_no, "expect",
|
||||
".expect() outside tests - use ? or annotate previous "
|
||||
"line with // SAFE-EXPECT: <reason>",
|
||||
)
|
||||
|
||||
# todo!/unimplemented!/unreachable!/unreachable_unchecked!
|
||||
if RE_PLACEHOLDER.search(code_only):
|
||||
report(
|
||||
str(file), line_no, "placeholder-macro",
|
||||
"todo!/unimplemented!/unreachable! in committed code",
|
||||
)
|
||||
|
||||
# Box<dyn Error>
|
||||
if RE_BOX_DYN_ERROR.search(code_only):
|
||||
report(
|
||||
str(file), line_no, "box-dyn-error",
|
||||
"Box<dyn Error> in non-test code - use anyhow::Error (apps) "
|
||||
"or thiserror enum (libs)",
|
||||
)
|
||||
|
||||
# panic!() in library code
|
||||
if is_lib_path(file) and RE_PANIC.search(code_only):
|
||||
report(
|
||||
str(file), line_no, "lib-panic",
|
||||
"panic!() in library code - return Result",
|
||||
)
|
||||
|
||||
# unsafe { without // SAFETY: — always enforced, even in tests
|
||||
if RE_UNSAFE_BLOCK.search(code_only):
|
||||
start = max(0, line_no_0 - 5)
|
||||
window = "\n".join(lines[start : line_no_0 + 1])
|
||||
if not RE_SAFETY.search(window):
|
||||
report(
|
||||
str(file), line_no, "unsafe-no-safety-comment",
|
||||
"unsafe block without // SAFETY: comment in preceding 5 lines",
|
||||
)
|
||||
|
||||
# #[allow(clippy::...)] without // CLIPPY-ALLOW: — always enforced
|
||||
if RE_CLIPPY_ALLOW.search(code_only):
|
||||
prev = lines[line_no_0 - 1] if line_no_0 > 0 else ""
|
||||
if not RE_CLIPPY_ALLOW_JUST.search(prev):
|
||||
report(
|
||||
str(file), line_no, "unjustified-clippy-allow",
|
||||
"#[allow(clippy::...)] without // CLIPPY-ALLOW: <reason> on "
|
||||
"previous line",
|
||||
)
|
||||
|
||||
# Narrowing numeric `as` casts
|
||||
if RE_NARROWING_CAST.search(code_only):
|
||||
report(
|
||||
str(file), line_no, "narrowing-as-cast",
|
||||
"possible narrowing 'as' cast - use TryFrom / try_into() for "
|
||||
"fallible conversion",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main() -> None:
|
||||
global violations
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print(f"Usage: {sys.argv[0]} <file.rs|dir> [file.rs|dir ...]", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
files = collect_rs_files(sys.argv[1:])
|
||||
if not files:
|
||||
print("warning: no .rs files found in the given arguments", file=sys.stderr)
|
||||
sys.exit(0)
|
||||
|
||||
for f in files:
|
||||
check_file(f)
|
||||
|
||||
if violations > 0:
|
||||
print("", file=sys.stderr)
|
||||
print(
|
||||
f"rust-programmer: {violations} violation(s). Fix before declaring work done.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print("", file=sys.stderr)
|
||||
print("Then run the full toolchain gate:", file=sys.stderr)
|
||||
print(" cargo +stable fmt --all -- --check", file=sys.stderr)
|
||||
print(
|
||||
" cargo +stable clippy --all-targets --all-features -- -D warnings",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(" cargo nextest run --all-targets --all-features", file=sys.stderr)
|
||||
print(
|
||||
" cargo +nightly miri nextest run --all-features # if unsafe touched",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(" cargo machete", file=sys.stderr)
|
||||
print(" cargo deny check", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"rust-programmer: no-excuse rules passed for {len(files)} file(s).")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env bash
|
||||
# No-excuse rule checker for Rust files.
|
||||
# Mirrors the philosophy of python-programmer / typescript-programmer scripts:
|
||||
# only rules that can be enforced via pure text matching live here.
|
||||
# Everything semantic is on clippy + miri + nextest.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "Usage: $0 <file.rs> [file.rs ...]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
violations=0
|
||||
report() {
|
||||
local file="$1"
|
||||
local line="$2"
|
||||
local rule="$3"
|
||||
local detail="$4"
|
||||
echo "::error file=${file},line=${line}::[${rule}] ${detail}" >&2
|
||||
violations=$((violations + 1))
|
||||
}
|
||||
|
||||
is_test_path() {
|
||||
local path="$1"
|
||||
case "$path" in
|
||||
*/tests/*|*/benches/*|*/examples/*|*/build.rs|*_test.rs|tests/*|benches/*|examples/*) return 0 ;;
|
||||
esac
|
||||
# In-file #[cfg(test)] modules are handled per-line below.
|
||||
return 1
|
||||
}
|
||||
|
||||
for file in "$@"; do
|
||||
[ -f "$file" ] || continue
|
||||
case "$file" in
|
||||
*.rs) ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
|
||||
if is_test_path "$file"; then
|
||||
# Test files are exempt from unwrap/expect/todo rules.
|
||||
# Still enforce unsafe-comment, allow-comment, panic-in-lib rules below
|
||||
# by setting a marker - keeping the loop unified.
|
||||
in_test_file=1
|
||||
else
|
||||
in_test_file=0
|
||||
fi
|
||||
|
||||
# Track #[cfg(test)] regions for per-line exemptions.
|
||||
in_cfg_test=0
|
||||
cfg_test_brace_depth=0
|
||||
line_no=0
|
||||
|
||||
while IFS= read -r raw_line || [ -n "$raw_line" ]; do
|
||||
line_no=$((line_no + 1))
|
||||
line="$raw_line"
|
||||
|
||||
# Crude #[cfg(test)] region tracker: when we see #[cfg(test)] on a
|
||||
# line followed by a mod with `{`, count braces until depth returns
|
||||
# to zero. This is approximate but matches typical formatting.
|
||||
if [[ "$line" =~ \#\[cfg\(test\)\] ]]; then
|
||||
in_cfg_test=1
|
||||
cfg_test_brace_depth=0
|
||||
fi
|
||||
if [ "$in_cfg_test" -eq 1 ]; then
|
||||
opens=$(printf '%s' "$line" | tr -cd '{' | wc -c)
|
||||
closes=$(printf '%s' "$line" | tr -cd '}' | wc -c)
|
||||
cfg_test_brace_depth=$((cfg_test_brace_depth + opens - closes))
|
||||
if [ "$cfg_test_brace_depth" -le 0 ] && [[ ! "$line" =~ \#\[cfg\(test\)\] ]]; then
|
||||
in_cfg_test=0
|
||||
fi
|
||||
fi
|
||||
|
||||
exempt=0
|
||||
[ "$in_test_file" -eq 1 ] && exempt=1
|
||||
[ "$in_cfg_test" -eq 1 ] && exempt=1
|
||||
|
||||
# Strip line comments before pattern checks - so doc comments and
|
||||
# explanatory prose do not trip the regexes.
|
||||
code_only="${line%%//*}"
|
||||
|
||||
if [ "$exempt" -eq 0 ]; then
|
||||
# .unwrap()
|
||||
if [[ "$code_only" =~ \.unwrap\(\) ]]; then
|
||||
# Allow if previous line had // SAFE-UNWRAP: comment
|
||||
prev_line=$(sed -n "$((line_no - 1))p" "$file" 2>/dev/null || true)
|
||||
if [[ ! "$prev_line" =~ //[[:space:]]*SAFE-UNWRAP: ]]; then
|
||||
report "$file" "$line_no" "unwrap" ".unwrap() outside tests - use ? / ok_or / pattern match or annotate previous line with // SAFE-UNWRAP: <reason>"
|
||||
fi
|
||||
fi
|
||||
|
||||
# .expect("...")
|
||||
if [[ "$code_only" =~ \.expect\( ]]; then
|
||||
prev_line=$(sed -n "$((line_no - 1))p" "$file" 2>/dev/null || true)
|
||||
if [[ ! "$prev_line" =~ //[[:space:]]*SAFE-EXPECT: ]]; then
|
||||
report "$file" "$line_no" "expect" ".expect() outside tests - use ? or annotate previous line with // SAFE-EXPECT: <reason>"
|
||||
fi
|
||||
fi
|
||||
|
||||
# todo!() / unimplemented!() / unreachable!()
|
||||
if [[ "$code_only" =~ (todo!|unimplemented!|unreachable!|unreachable_unchecked!) ]]; then
|
||||
report "$file" "$line_no" "placeholder-macro" "todo!/unimplemented!/unreachable! in committed code"
|
||||
fi
|
||||
|
||||
# Box<dyn Error
|
||||
if [[ "$code_only" =~ Box\<dyn[[:space:]]+Error ]]; then
|
||||
report "$file" "$line_no" "box-dyn-error" "Box<dyn Error> in non-test code - use anyhow::Error (apps) or thiserror enum (libs)"
|
||||
fi
|
||||
|
||||
# panic!( in lib
|
||||
if [[ "$file" == */src/lib.rs || "$file" == */src/*/mod.rs || ( "$file" == */src/*.rs && "$file" != */src/main.rs && "$file" != */src/bin/* ) ]]; then
|
||||
if [[ "$code_only" =~ panic!\( ]]; then
|
||||
report "$file" "$line_no" "lib-panic" "panic!() in library code - return Result"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# unsafe { without preceding // SAFETY: in the last 5 lines (always enforced)
|
||||
if [[ "$code_only" =~ unsafe[[:space:]]*\{ ]]; then
|
||||
start=$((line_no > 5 ? line_no - 5 : 1))
|
||||
window=$(sed -n "${start},${line_no}p" "$file" 2>/dev/null || true)
|
||||
if [[ ! "$window" =~ //[[:space:]]*SAFETY: ]]; then
|
||||
report "$file" "$line_no" "unsafe-no-safety-comment" "unsafe block without // SAFETY: comment in preceding 5 lines"
|
||||
fi
|
||||
fi
|
||||
|
||||
# #[allow(clippy::...)] without preceding // CLIPPY-ALLOW: justification
|
||||
if [[ "$code_only" =~ \#\[allow\(clippy:: ]]; then
|
||||
prev_line=$(sed -n "$((line_no - 1))p" "$file" 2>/dev/null || true)
|
||||
if [[ ! "$prev_line" =~ //[[:space:]]*CLIPPY-ALLOW: ]]; then
|
||||
report "$file" "$line_no" "unjustified-clippy-allow" "#[allow(clippy::...)] without // CLIPPY-ALLOW: <reason> on previous line"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Narrowing numeric `as` casts - heuristic flag for human review.
|
||||
# Catches the common shapes; precise type analysis belongs to clippy::cast_possible_truncation.
|
||||
if [[ "$code_only" =~ as[[:space:]]+(u8|u16|u32|i8|i16|i32) ]] && \
|
||||
[[ "$code_only" =~ (u16|u32|u64|u128|usize|i16|i32|i64|i128|isize)[[:space:]]+as[[:space:]]+(u8|u16|u32|i8|i16|i32) ]]; then
|
||||
report "$file" "$line_no" "narrowing-as-cast" "possible narrowing 'as' cast - use TryFrom / try_into() for fallible conversion"
|
||||
fi
|
||||
done < "$file"
|
||||
done
|
||||
|
||||
if [ "$violations" -gt 0 ]; then
|
||||
echo "" >&2
|
||||
echo "rust-programmer: ${violations} violation(s). Fix before declaring work done." >&2
|
||||
echo "" >&2
|
||||
echo "Then run the full toolchain gate:" >&2
|
||||
echo " cargo +stable fmt --all -- --check" >&2
|
||||
echo " cargo +stable clippy --all-targets --all-features -- -D warnings" >&2
|
||||
echo " cargo nextest run --all-targets --all-features" >&2
|
||||
echo " cargo +nightly miri nextest run --all-features # if unsafe touched" >&2
|
||||
echo " cargo machete" >&2
|
||||
echo " cargo deny check" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "rust-programmer: no-excuse rules passed for $# file(s)."
|
||||
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# dependencies = [
|
||||
# "typer",
|
||||
# "rich",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
# ─── How to run ───
|
||||
# 1. Install uv (if not installed):
|
||||
# curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
# 2. Run:
|
||||
# uv run new-project.py myproject
|
||||
# uv run new-project.py myproject --path ./workspace
|
||||
# ──────────────────
|
||||
#
|
||||
# Creates a new Rust project with strict lints, deny.toml, rustfmt.toml,
|
||||
# rust-toolchain.toml, and .cargo/config.toml pre-configured.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
|
||||
console = Console(stderr=True)
|
||||
|
||||
# ── Embedded config contents ─────────────────────────────────────────────
|
||||
|
||||
RUST_TOOLCHAIN_TOML = """\
|
||||
[toolchain]
|
||||
channel = "stable"
|
||||
components = ["rustfmt", "clippy", "rust-src"]
|
||||
profile = "default"
|
||||
"""
|
||||
|
||||
CARGO_TOML_LINTS = """
|
||||
[lints.rust]
|
||||
unsafe_op_in_unsafe_fn = "deny"
|
||||
missing_docs = "warn"
|
||||
missing_debug_implementations = "warn"
|
||||
unreachable_pub = "warn"
|
||||
unused_must_use = "deny"
|
||||
elided_lifetimes_in_paths = "warn"
|
||||
non_ascii_idents = "deny"
|
||||
trivial_numeric_casts = "warn"
|
||||
unused_lifetimes = "warn"
|
||||
single_use_lifetimes = "warn"
|
||||
|
||||
[lints.clippy]
|
||||
all = { level = "deny", priority = -1 }
|
||||
pedantic = { level = "warn", priority = -1 }
|
||||
nursery = { level = "warn", priority = -1 }
|
||||
cargo = { level = "warn", priority = -1 }
|
||||
undocumented_unsafe_blocks = "deny"
|
||||
multiple_unsafe_ops_per_block = "deny"
|
||||
unwrap_used = "deny"
|
||||
expect_used = "deny"
|
||||
panic = "deny"
|
||||
todo = "deny"
|
||||
unimplemented = "deny"
|
||||
dbg_macro = "deny"
|
||||
print_stdout = "warn"
|
||||
print_stderr = "warn"
|
||||
module_name_repetitions = { level = "allow" }
|
||||
must_use_candidate = { level = "allow" }
|
||||
missing_errors_doc = { level = "allow" }
|
||||
missing_panics_doc = { level = "allow" }
|
||||
"""
|
||||
|
||||
CARGO_CONFIG_TOML = """\
|
||||
[build]
|
||||
rustflags = ["-C", "link-arg=-fuse-ld=lld"]
|
||||
|
||||
[target.x86_64-unknown-linux-gnu]
|
||||
linker = "clang"
|
||||
rustflags = ["-C", "link-arg=-fuse-ld=lld"]
|
||||
|
||||
[target.aarch64-apple-darwin]
|
||||
rustflags = []
|
||||
"""
|
||||
|
||||
DENY_TOML = """\
|
||||
[advisories]
|
||||
vulnerability = "deny"
|
||||
unmaintained = "warn"
|
||||
yanked = "deny"
|
||||
|
||||
[licenses]
|
||||
unlicensed = "deny"
|
||||
allow = ["MIT", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "ISC", "Unicode-3.0", "Zlib"]
|
||||
|
||||
[bans]
|
||||
multiple-versions = "warn"
|
||||
wildcards = "deny"
|
||||
|
||||
[sources]
|
||||
unknown-registry = "deny"
|
||||
unknown-git = "deny"
|
||||
"""
|
||||
|
||||
RUSTFMT_TOML = """\
|
||||
edition = "2024"
|
||||
max_width = 100
|
||||
use_field_init_shorthand = true
|
||||
use_try_shorthand = true
|
||||
"""
|
||||
|
||||
# ── Main ─────────────────────────────────────────────────────────────────
|
||||
|
||||
app = typer.Typer(add_completion=False)
|
||||
|
||||
|
||||
@app.command()
|
||||
def main(
|
||||
name: str = typer.Argument(help="Name of the new Rust project"),
|
||||
path: Path = typer.Option(
|
||||
Path.cwd(),
|
||||
"--path",
|
||||
"-p",
|
||||
help="Parent directory where the project folder is created",
|
||||
),
|
||||
) -> None:
|
||||
"""Scaffold a new Rust project with strict lints and tooling configs."""
|
||||
project_dir = path / name
|
||||
|
||||
# ── cargo init ───────────────────────────────────────────────────
|
||||
console.print(f"[bold green]Creating[/] project [cyan]{name}[/] at [dim]{project_dir}[/]")
|
||||
try:
|
||||
subprocess.run(
|
||||
["cargo", "init", str(project_dir), "--name", name],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
console.print("[bold red]Error:[/] cargo not found. Install Rust via https://rustup.rs")
|
||||
sys.exit(1)
|
||||
except subprocess.CalledProcessError as exc:
|
||||
console.print(f"[bold red]cargo init failed:[/]\n{exc.stderr}")
|
||||
sys.exit(1)
|
||||
|
||||
# ── rust-toolchain.toml ──────────────────────────────────────────
|
||||
(project_dir / "rust-toolchain.toml").write_text(RUST_TOOLCHAIN_TOML)
|
||||
console.print(" [dim]wrote[/] rust-toolchain.toml")
|
||||
|
||||
# ── Append [lints] to Cargo.toml ─────────────────────────────────
|
||||
cargo_toml = project_dir / "Cargo.toml"
|
||||
with cargo_toml.open("a") as f:
|
||||
f.write(CARGO_TOML_LINTS)
|
||||
console.print(" [dim]appended[/] [lints] to Cargo.toml")
|
||||
|
||||
# ── .cargo/config.toml ───────────────────────────────────────────
|
||||
cargo_config_dir = project_dir / ".cargo"
|
||||
cargo_config_dir.mkdir(parents=True, exist_ok=True)
|
||||
(cargo_config_dir / "config.toml").write_text(CARGO_CONFIG_TOML)
|
||||
console.print(" [dim]wrote[/] .cargo/config.toml")
|
||||
|
||||
# ── deny.toml ────────────────────────────────────────────────────
|
||||
(project_dir / "deny.toml").write_text(DENY_TOML)
|
||||
console.print(" [dim]wrote[/] deny.toml")
|
||||
|
||||
# ── rustfmt.toml ─────────────────────────────────────────────────
|
||||
(project_dir / "rustfmt.toml").write_text(RUSTFMT_TOML)
|
||||
console.print(" [dim]wrote[/] rustfmt.toml")
|
||||
|
||||
console.print(f"\n[bold green]Done![/] cd {project_dir} && cargo check")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user