From 53de295b21746b1bf9720263dc658ee3037b0e95 Mon Sep 17 00:00:00 2001 From: YeonGyu-Kim Date: Fri, 15 May 2026 11:41:35 +0900 Subject: [PATCH] perf(ci): add sharded test runner phases Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- script/run-ci-tests.test.ts | 34 ++++++++++ script/run-ci-tests.ts | 120 +++++++++++++++++++++++++++++++++--- 2 files changed, 144 insertions(+), 10 deletions(-) diff --git a/script/run-ci-tests.test.ts b/script/run-ci-tests.test.ts index 22ad433b2..0f54f4e81 100644 --- a/script/run-ci-tests.test.ts +++ b/script/run-ci-tests.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "bun:test" +import { selectCiTestTargets } from "./run-ci-tests" describe("test script isolation", () => { test("#given mock.module tests in the suite #then bun run test uses the isolated CI runner", async () => { @@ -8,4 +9,37 @@ describe("test script isolation", () => { //#then expect(packageJson.scripts.test).toBe("bun run script/run-ci-tests.ts") }) + + test("#given isolated test shards #when selecting targets #then shards are deterministic and complete", () => { + // given + const ciTestPlan = { + isolatedModuleMockFiles: [], + isolatedTestTargets: ["a.test.ts", "b.test.ts", "c.test.ts", "d.test.ts", "e.test.ts"], + sharedTestFiles: ["shared.test.ts"], + } + + // when + const shardOne = selectCiTestTargets(ciTestPlan, { phase: "isolated", shardCount: 2, shardIndex: 0 }) + const shardTwo = selectCiTestTargets(ciTestPlan, { phase: "isolated", shardCount: 2, shardIndex: 1 }) + + // then + expect(shardOne).toEqual({ isolatedTestTargets: ["a.test.ts", "c.test.ts", "e.test.ts"], sharedTestFiles: [] }) + expect(shardTwo).toEqual({ isolatedTestTargets: ["b.test.ts", "d.test.ts"], sharedTestFiles: [] }) + expect([...shardOne.isolatedTestTargets, ...shardTwo.isolatedTestTargets].sort()).toEqual(ciTestPlan.isolatedTestTargets) + }) + + test("#given shared phase #when selecting targets #then only shared tests run", () => { + // given + const ciTestPlan = { + isolatedModuleMockFiles: [], + isolatedTestTargets: ["isolated.test.ts"], + sharedTestFiles: ["shared.test.ts"], + } + + // when + const selectedTargets = selectCiTestTargets(ciTestPlan, { phase: "shared", shardCount: 1, shardIndex: 0 }) + + // then + expect(selectedTargets).toEqual({ isolatedTestTargets: [], sharedTestFiles: ["shared.test.ts"] }) + }) }) diff --git a/script/run-ci-tests.ts b/script/run-ci-tests.ts index cae400858..1c77bc627 100644 --- a/script/run-ci-tests.ts +++ b/script/run-ci-tests.ts @@ -6,6 +6,19 @@ type CiTestPlan = { sharedTestFiles: string[] } +type CiTestPhase = "all" | "isolated" | "shared" + +type CiTestRunOptions = { + phase: CiTestPhase + shardCount: number + shardIndex: number +} + +type CiTestTargetSelection = { + isolatedTestTargets: string[] + sharedTestFiles: string[] +} + const TEST_ROOTS = ["bin", "script", "src"] as const const MODULE_MOCK_PATTERN = "mock.module(" const ALWAYS_ISOLATED_TEST_FILES = [ @@ -62,6 +75,86 @@ function collapseNestedTargets(isolatedTargets: string[]): string[] { }) } +function readFlagValue(args: string[], flagName: string): string | null { + const prefix = `${flagName}=` + const flag = args.find((arg) => arg.startsWith(prefix)) + + return flag?.slice(prefix.length) ?? null +} + +function parsePhase(rawPhase: string | null): CiTestPhase { + if (rawPhase === null) { + return "all" + } + + if (rawPhase === "all" || rawPhase === "isolated" || rawPhase === "shared") { + return rawPhase + } + + throw new Error(`Invalid --phase value: ${rawPhase}. Expected all, isolated, or shared.`) +} + +function parsePositiveIntegerFlag(args: string[], flagName: string, defaultValue: number): number { + const rawValue = readFlagValue(args, flagName) + if (rawValue === null) { + return defaultValue + } + + const parsedValue = Number(rawValue) + if (!Number.isInteger(parsedValue) || parsedValue < 1) { + throw new Error(`Invalid ${flagName} value: ${rawValue}. Expected a positive integer.`) + } + + return parsedValue +} + +function parseNonNegativeIntegerFlag(args: string[], flagName: string, defaultValue: number): number { + const rawValue = readFlagValue(args, flagName) + if (rawValue === null) { + return defaultValue + } + + const parsedValue = Number(rawValue) + if (!Number.isInteger(parsedValue) || parsedValue < 0) { + throw new Error(`Invalid ${flagName} value: ${rawValue}. Expected a non-negative integer.`) + } + + return parsedValue +} + +function parseCiTestRunOptions(args: string[]): CiTestRunOptions { + const phase = parsePhase(readFlagValue(args, "--phase")) + const shardCount = parsePositiveIntegerFlag(args, "--shard-count", 1) + const shardIndex = parseNonNegativeIntegerFlag(args, "--shard-index", 0) + + if (shardIndex >= shardCount) { + throw new Error(`Invalid --shard-index value: ${shardIndex}. Expected a value less than --shard-count ${shardCount}.`) + } + + if (shardCount > 1 && phase !== "isolated") { + throw new Error("Test sharding is only supported with --phase=isolated.") + } + + return { phase, shardCount, shardIndex } +} + +function selectShard(testTargets: string[], shardCount: number, shardIndex: number): string[] { + if (shardCount === 1) { + return testTargets + } + + return testTargets.filter((_, index) => index % shardCount === shardIndex) +} + +export function selectCiTestTargets(ciTestPlan: CiTestPlan, options: CiTestRunOptions): CiTestTargetSelection { + const isolatedTestTargets = options.phase === "shared" + ? [] + : selectShard(ciTestPlan.isolatedTestTargets, options.shardCount, options.shardIndex) + const sharedTestFiles = options.phase === "isolated" ? [] : ciTestPlan.sharedTestFiles + + return { isolatedTestTargets, sharedTestFiles } +} + export async function createCiTestPlan(rootDirectory: string = process.cwd()): Promise { const allTestFiles = await collectTestFiles(rootDirectory) const isolatedModuleMockFiles: string[] = [] @@ -97,16 +190,15 @@ async function runBunTest(testFiles: string[], label: string): Promise { } console.log(`::group::${label}`) - - // For directory paths, exclude _auc* directories which are separate isolated targets - const args = testFiles.map(tf => { - if (tf.includes('/') && !tf.endsWith('.test.ts')) { - // It's a directory path, add negation glob - return [tf, '!_auc-*/**/*.test.ts'] + + const args = testFiles.map((testFile) => { + if (testFile.includes("/") && !testFile.endsWith(".test.ts")) { + return [testFile, "!_auc-*/**/*.test.ts"] } - return tf + + return testFile }).flat() - + const command = ["bun", "test", ...args] const spawnedProcess = Bun.spawn(command, { cwd: process.cwd(), @@ -123,17 +215,25 @@ async function runBunTest(testFiles: string[], label: string): Promise { } async function main(): Promise { + const options = parseCiTestRunOptions(process.argv.slice(2)) const ciTestPlan = await createCiTestPlan() + const selectedTargets = selectCiTestTargets(ciTestPlan, options) console.log( `Detected ${ciTestPlan.isolatedModuleMockFiles.length} mock.module() test files, ${ciTestPlan.isolatedTestTargets.length} isolated targets, and ${ciTestPlan.sharedTestFiles.length} shared test files.`, ) - for (const isolatedTestTarget of ciTestPlan.isolatedTestTargets) { + if (options.phase === "isolated" && options.shardCount > 1) { + console.log( + `Running isolated test shard ${options.shardIndex + 1}/${options.shardCount} with ${selectedTargets.isolatedTestTargets.length} targets.`, + ) + } + + for (const isolatedTestTarget of selectedTargets.isolatedTestTargets) { await runBunTest([isolatedTestTarget], `Isolated ${isolatedTestTarget}`) } - await runBunTest(ciTestPlan.sharedTestFiles, "Shared Bun test suite") + await runBunTest(selectedTargets.sharedTestFiles, "Shared Bun test suite") } export const moduleMockPattern = MODULE_MOCK_PATTERN