From 53de295b21746b1bf9720263dc658ee3037b0e95 Mon Sep 17 00:00:00 2001 From: YeonGyu-Kim Date: Fri, 15 May 2026 11:41:35 +0900 Subject: [PATCH 1/3] perf(ci): add sharded test runner phases Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- script/run-ci-tests.test.ts | 34 ++++++++++ script/run-ci-tests.ts | 120 +++++++++++++++++++++++++++++++++--- 2 files changed, 144 insertions(+), 10 deletions(-) diff --git a/script/run-ci-tests.test.ts b/script/run-ci-tests.test.ts index 22ad433b2..0f54f4e81 100644 --- a/script/run-ci-tests.test.ts +++ b/script/run-ci-tests.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "bun:test" +import { selectCiTestTargets } from "./run-ci-tests" describe("test script isolation", () => { test("#given mock.module tests in the suite #then bun run test uses the isolated CI runner", async () => { @@ -8,4 +9,37 @@ describe("test script isolation", () => { //#then expect(packageJson.scripts.test).toBe("bun run script/run-ci-tests.ts") }) + + test("#given isolated test shards #when selecting targets #then shards are deterministic and complete", () => { + // given + const ciTestPlan = { + isolatedModuleMockFiles: [], + isolatedTestTargets: ["a.test.ts", "b.test.ts", "c.test.ts", "d.test.ts", "e.test.ts"], + sharedTestFiles: ["shared.test.ts"], + } + + // when + const shardOne = selectCiTestTargets(ciTestPlan, { phase: "isolated", shardCount: 2, shardIndex: 0 }) + const shardTwo = selectCiTestTargets(ciTestPlan, { phase: "isolated", shardCount: 2, shardIndex: 1 }) + + // then + expect(shardOne).toEqual({ isolatedTestTargets: ["a.test.ts", "c.test.ts", "e.test.ts"], sharedTestFiles: [] }) + expect(shardTwo).toEqual({ isolatedTestTargets: ["b.test.ts", "d.test.ts"], sharedTestFiles: [] }) + expect([...shardOne.isolatedTestTargets, ...shardTwo.isolatedTestTargets].sort()).toEqual(ciTestPlan.isolatedTestTargets) + }) + + test("#given shared phase #when selecting targets #then only shared tests run", () => { + // given + const ciTestPlan = { + isolatedModuleMockFiles: [], + isolatedTestTargets: ["isolated.test.ts"], + sharedTestFiles: ["shared.test.ts"], + } + + // when + const selectedTargets = selectCiTestTargets(ciTestPlan, { phase: "shared", shardCount: 1, shardIndex: 0 }) + + // then + expect(selectedTargets).toEqual({ isolatedTestTargets: [], sharedTestFiles: ["shared.test.ts"] }) + }) }) diff --git a/script/run-ci-tests.ts b/script/run-ci-tests.ts index cae400858..1c77bc627 100644 --- a/script/run-ci-tests.ts +++ b/script/run-ci-tests.ts @@ -6,6 +6,19 @@ type CiTestPlan = { sharedTestFiles: string[] } +type CiTestPhase = "all" | "isolated" | "shared" + +type CiTestRunOptions = { + phase: CiTestPhase + shardCount: number + shardIndex: number +} + +type CiTestTargetSelection = { + isolatedTestTargets: string[] + sharedTestFiles: string[] +} + const TEST_ROOTS = ["bin", "script", "src"] as const const MODULE_MOCK_PATTERN = "mock.module(" const ALWAYS_ISOLATED_TEST_FILES = [ @@ -62,6 +75,86 @@ function collapseNestedTargets(isolatedTargets: string[]): string[] { }) } +function readFlagValue(args: string[], flagName: string): string | null { + const prefix = `${flagName}=` + const flag = args.find((arg) => arg.startsWith(prefix)) + + return flag?.slice(prefix.length) ?? null +} + +function parsePhase(rawPhase: string | null): CiTestPhase { + if (rawPhase === null) { + return "all" + } + + if (rawPhase === "all" || rawPhase === "isolated" || rawPhase === "shared") { + return rawPhase + } + + throw new Error(`Invalid --phase value: ${rawPhase}. Expected all, isolated, or shared.`) +} + +function parsePositiveIntegerFlag(args: string[], flagName: string, defaultValue: number): number { + const rawValue = readFlagValue(args, flagName) + if (rawValue === null) { + return defaultValue + } + + const parsedValue = Number(rawValue) + if (!Number.isInteger(parsedValue) || parsedValue < 1) { + throw new Error(`Invalid ${flagName} value: ${rawValue}. Expected a positive integer.`) + } + + return parsedValue +} + +function parseNonNegativeIntegerFlag(args: string[], flagName: string, defaultValue: number): number { + const rawValue = readFlagValue(args, flagName) + if (rawValue === null) { + return defaultValue + } + + const parsedValue = Number(rawValue) + if (!Number.isInteger(parsedValue) || parsedValue < 0) { + throw new Error(`Invalid ${flagName} value: ${rawValue}. Expected a non-negative integer.`) + } + + return parsedValue +} + +function parseCiTestRunOptions(args: string[]): CiTestRunOptions { + const phase = parsePhase(readFlagValue(args, "--phase")) + const shardCount = parsePositiveIntegerFlag(args, "--shard-count", 1) + const shardIndex = parseNonNegativeIntegerFlag(args, "--shard-index", 0) + + if (shardIndex >= shardCount) { + throw new Error(`Invalid --shard-index value: ${shardIndex}. Expected a value less than --shard-count ${shardCount}.`) + } + + if (shardCount > 1 && phase !== "isolated") { + throw new Error("Test sharding is only supported with --phase=isolated.") + } + + return { phase, shardCount, shardIndex } +} + +function selectShard(testTargets: string[], shardCount: number, shardIndex: number): string[] { + if (shardCount === 1) { + return testTargets + } + + return testTargets.filter((_, index) => index % shardCount === shardIndex) +} + +export function selectCiTestTargets(ciTestPlan: CiTestPlan, options: CiTestRunOptions): CiTestTargetSelection { + const isolatedTestTargets = options.phase === "shared" + ? [] + : selectShard(ciTestPlan.isolatedTestTargets, options.shardCount, options.shardIndex) + const sharedTestFiles = options.phase === "isolated" ? [] : ciTestPlan.sharedTestFiles + + return { isolatedTestTargets, sharedTestFiles } +} + export async function createCiTestPlan(rootDirectory: string = process.cwd()): Promise { const allTestFiles = await collectTestFiles(rootDirectory) const isolatedModuleMockFiles: string[] = [] @@ -97,16 +190,15 @@ async function runBunTest(testFiles: string[], label: string): Promise { } console.log(`::group::${label}`) - - // For directory paths, exclude _auc* directories which are separate isolated targets - const args = testFiles.map(tf => { - if (tf.includes('/') && !tf.endsWith('.test.ts')) { - // It's a directory path, add negation glob - return [tf, '!_auc-*/**/*.test.ts'] + + const args = testFiles.map((testFile) => { + if (testFile.includes("/") && !testFile.endsWith(".test.ts")) { + return [testFile, "!_auc-*/**/*.test.ts"] } - return tf + + return testFile }).flat() - + const command = ["bun", "test", ...args] const spawnedProcess = Bun.spawn(command, { cwd: process.cwd(), @@ -123,17 +215,25 @@ async function runBunTest(testFiles: string[], label: string): Promise { } async function main(): Promise { + const options = parseCiTestRunOptions(process.argv.slice(2)) const ciTestPlan = await createCiTestPlan() + const selectedTargets = selectCiTestTargets(ciTestPlan, options) console.log( `Detected ${ciTestPlan.isolatedModuleMockFiles.length} mock.module() test files, ${ciTestPlan.isolatedTestTargets.length} isolated targets, and ${ciTestPlan.sharedTestFiles.length} shared test files.`, ) - for (const isolatedTestTarget of ciTestPlan.isolatedTestTargets) { + if (options.phase === "isolated" && options.shardCount > 1) { + console.log( + `Running isolated test shard ${options.shardIndex + 1}/${options.shardCount} with ${selectedTargets.isolatedTestTargets.length} targets.`, + ) + } + + for (const isolatedTestTarget of selectedTargets.isolatedTestTargets) { await runBunTest([isolatedTestTarget], `Isolated ${isolatedTestTarget}`) } - await runBunTest(ciTestPlan.sharedTestFiles, "Shared Bun test suite") + await runBunTest(selectedTargets.sharedTestFiles, "Shared Bun test suite") } export const moduleMockPattern = MODULE_MOCK_PATTERN From fdd40815ba006195bfbf74f8ad0b8933516ff6d2 Mon Sep 17 00:00:00 2001 From: YeonGyu-Kim Date: Fri, 15 May 2026 11:41:35 +0900 Subject: [PATCH 2/3] ci: split test workflow across shards Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .github/workflows/ci.yml | 58 ++++++++++++++++++++++++++++++--- script/publish-workflow.test.ts | 25 ++++++++++---- 2 files changed, 71 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44b17367f..7c0975700 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,34 @@ jobs: echo "PR targets '${BASE_REF}' branch - OK" fi - test: + test-isolated: + name: Isolated tests (${{ matrix.shard }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3] + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.3.12" + + - uses: actions/cache@v4 + with: + path: ~/.bun/install/cache + key: ${{ runner.os }}-bun-1.3.12-${{ hashFiles('bun.lock') }} + + - name: Install dependencies + run: bun install --frozen-lockfile + env: + BUN_INSTALL_ALLOW_SCRIPTS: "@ast-grep/napi" + + - name: Run isolated tests + run: bun run script/run-ci-tests.ts --phase=isolated --shard-count=4 --shard-index=${{ matrix.shard }} + + test-shared: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -41,16 +68,24 @@ jobs: with: bun-version: "1.3.12" + - uses: actions/cache@v4 + with: + path: ~/.bun/install/cache + key: ${{ runner.os }}-bun-1.3.12-${{ hashFiles('bun.lock') }} + - name: Install dependencies run: bun install --frozen-lockfile env: BUN_INSTALL_ALLOW_SCRIPTS: "@ast-grep/napi" - - name: Build plugin - run: bun run build + - name: Run shared tests + run: bun run script/run-ci-tests.ts --phase=shared - - name: Run tests - run: bun run script/run-ci-tests.ts + test: + runs-on: ubuntu-latest + needs: [test-isolated, test-shared] + steps: + - run: echo "All test shards passed" typecheck: runs-on: ubuntu-latest @@ -61,6 +96,11 @@ jobs: with: bun-version: "1.3.12" + - uses: actions/cache@v4 + with: + path: ~/.bun/install/cache + key: ${{ runner.os }}-bun-1.3.12-${{ hashFiles('bun.lock') }} + - name: Install dependencies run: bun install --frozen-lockfile env: @@ -86,6 +126,11 @@ jobs: with: bun-version: "1.3.12" + - uses: actions/cache@v4 + with: + path: ~/.bun/install/cache + key: ${{ runner.os }}-bun-1.3.12-${{ hashFiles('bun.lock') }} + - name: Install dependencies run: bun install --frozen-lockfile env: @@ -99,6 +144,9 @@ jobs: test -f dist/index.js || (echo "ERROR: dist/index.js not found!" && exit 1) test -f dist/index.d.ts || (echo "ERROR: dist/index.d.ts not found!" && exit 1) + - name: Verify dist bundle tests + run: bun test src/shared/dist-bundle-bun-globals.test.ts + - name: Auto-commit schema changes if: github.event_name == 'push' && github.ref == 'refs/heads/master' run: | diff --git a/script/publish-workflow.test.ts b/script/publish-workflow.test.ts index f1f45eb5b..70e97a40b 100644 --- a/script/publish-workflow.test.ts +++ b/script/publish-workflow.test.ts @@ -3,19 +3,30 @@ import { describe, expect, test } from "bun:test" import { readFileSync } from "node:fs" -const workflowPaths = [ - new URL("../.github/workflows/ci.yml", import.meta.url), - new URL("../.github/workflows/publish.yml", import.meta.url), +const workflowChecks = [ + { + path: new URL("../.github/workflows/ci.yml", import.meta.url), + testRuns: [ + "run: bun run script/run-ci-tests.ts --phase=isolated --shard-count=4 --shard-index=${{ matrix.shard }}", + "run: bun run script/run-ci-tests.ts --phase=shared", + "run: bun test src/shared/dist-bundle-bun-globals.test.ts", + ], + }, + { + path: new URL("../.github/workflows/publish.yml", import.meta.url), + testRuns: ["run: bun run script/run-ci-tests.ts"], + }, ] describe("test workflows", () => { test("use pure bun test for workflows", () => { - for (const workflowPath of workflowPaths) { + for (const workflowCheck of workflowChecks) { // #given - const workflow = readFileSync(workflowPath, "utf8") + const workflow = readFileSync(workflowCheck.path, "utf8") - expect(workflow).toContain("- name: Run tests") - expect(workflow).toMatch(/run: bun (test|run script\/run-ci-tests\.ts)/) + for (const testRun of workflowCheck.testRuns) { + expect(workflow).toContain(testRun) + } } }) }) From e80c2811b1b4498421e4e9f669d92e054616ebee Mon Sep 17 00:00:00 2001 From: YeonGyu-Kim Date: Fri, 15 May 2026 12:07:58 +0900 Subject: [PATCH 3/3] ci: fail closed on sharded test gate --- .github/workflows/ci.yml | 12 +++++++++++- script/publish-workflow.test.ts | 4 ++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c0975700..42dcb7695 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,8 +84,18 @@ jobs: test: runs-on: ubuntu-latest needs: [test-isolated, test-shared] + if: ${{ always() }} steps: - - run: echo "All test shards passed" + - name: Verify test shards passed + env: + ISOLATED_RESULT: ${{ needs.test-isolated.result }} + SHARED_RESULT: ${{ needs.test-shared.result }} + run: | + if [ "$ISOLATED_RESULT" != "success" ] || [ "$SHARED_RESULT" != "success" ]; then + echo "::error::test-isolated=${ISOLATED_RESULT}, test-shared=${SHARED_RESULT}" + exit 1 + fi + echo "All test shards passed" typecheck: runs-on: ubuntu-latest diff --git a/script/publish-workflow.test.ts b/script/publish-workflow.test.ts index 70e97a40b..c22921d4b 100644 --- a/script/publish-workflow.test.ts +++ b/script/publish-workflow.test.ts @@ -9,6 +9,10 @@ const workflowChecks = [ testRuns: [ "run: bun run script/run-ci-tests.ts --phase=isolated --shard-count=4 --shard-index=${{ matrix.shard }}", "run: bun run script/run-ci-tests.ts --phase=shared", + "if: ${{ always() }}", + "ISOLATED_RESULT: ${{ needs.test-isolated.result }}", + "SHARED_RESULT: ${{ needs.test-shared.result }}", + "echo \"::error::test-isolated=${ISOLATED_RESULT}, test-shared=${SHARED_RESULT}\"", "run: bun test src/shared/dist-bundle-bun-globals.test.ts", ], },