Merge pull request #4033 from code-yeongyu/perf/ci-test-build-time-20260515

perf(ci): shard CI test runner

.github/workflows/ci.yml

@@ -32,7 +32,34 @@ jobs:
             echo "PR targets '${BASE_REF}' branch - OK"
           fi
 
-  test:
+  test-isolated:
+    name: Isolated tests (${{ matrix.shard }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [0, 1, 2, 3]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: "1.3.12"
+
+      - uses: actions/cache@v4
+        with:
+          path: ~/.bun/install/cache
+          key: ${{ runner.os }}-bun-1.3.12-${{ hashFiles('bun.lock') }}
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+        env:
+          BUN_INSTALL_ALLOW_SCRIPTS: "@ast-grep/napi"
+
+      - name: Run isolated tests
+        run: bun run script/run-ci-tests.ts --phase=isolated --shard-count=4 --shard-index=${{ matrix.shard }}
+
+  test-shared:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -41,16 +68,34 @@ jobs:
         with:
           bun-version: "1.3.12"
 
+      - uses: actions/cache@v4
+        with:
+          path: ~/.bun/install/cache
+          key: ${{ runner.os }}-bun-1.3.12-${{ hashFiles('bun.lock') }}
+
       - name: Install dependencies
         run: bun install --frozen-lockfile
         env:
           BUN_INSTALL_ALLOW_SCRIPTS: "@ast-grep/napi"
 
-      - name: Build plugin
-        run: bun run build
+      - name: Run shared tests
+        run: bun run script/run-ci-tests.ts --phase=shared
 
-      - name: Run tests
-        run: bun run script/run-ci-tests.ts
+  test:
+    runs-on: ubuntu-latest
+    needs: [test-isolated, test-shared]
+    if: ${{ always() }}
+    steps:
+      - name: Verify test shards passed
+        env:
+          ISOLATED_RESULT: ${{ needs.test-isolated.result }}
+          SHARED_RESULT: ${{ needs.test-shared.result }}
+        run: |
+          if [ "$ISOLATED_RESULT" != "success" ] || [ "$SHARED_RESULT" != "success" ]; then
+            echo "::error::test-isolated=${ISOLATED_RESULT}, test-shared=${SHARED_RESULT}"
+            exit 1
+          fi
+          echo "All test shards passed"
 
   typecheck:
     runs-on: ubuntu-latest
@@ -61,6 +106,11 @@ jobs:
         with:
           bun-version: "1.3.12"
 
+      - uses: actions/cache@v4
+        with:
+          path: ~/.bun/install/cache
+          key: ${{ runner.os }}-bun-1.3.12-${{ hashFiles('bun.lock') }}
+
       - name: Install dependencies
         run: bun install --frozen-lockfile
         env:
@@ -86,6 +136,11 @@ jobs:
         with:
           bun-version: "1.3.12"
 
+      - uses: actions/cache@v4
+        with:
+          path: ~/.bun/install/cache
+          key: ${{ runner.os }}-bun-1.3.12-${{ hashFiles('bun.lock') }}
+
       - name: Install dependencies
         run: bun install --frozen-lockfile
         env:
@@ -99,6 +154,9 @@ jobs:
           test -f dist/index.js || (echo "ERROR: dist/index.js not found!" && exit 1)
           test -f dist/index.d.ts || (echo "ERROR: dist/index.d.ts not found!" && exit 1)
 
+      - name: Verify dist bundle tests
+        run: bun test src/shared/dist-bundle-bun-globals.test.ts
+
       - name: Auto-commit schema changes
         if: github.event_name == 'push' && github.ref == 'refs/heads/master'
         run: |

script/publish-workflow.test.ts

@@ -3,19 +3,34 @@
 import { describe, expect, test } from "bun:test"
 import { readFileSync } from "node:fs"
 
-const workflowPaths = [
-  new URL("../.github/workflows/ci.yml", import.meta.url),
-  new URL("../.github/workflows/publish.yml", import.meta.url),
+const workflowChecks = [
+  {
+    path: new URL("../.github/workflows/ci.yml", import.meta.url),
+    testRuns: [
+      "run: bun run script/run-ci-tests.ts --phase=isolated --shard-count=4 --shard-index=${{ matrix.shard }}",
+      "run: bun run script/run-ci-tests.ts --phase=shared",
+      "if: ${{ always() }}",
+      "ISOLATED_RESULT: ${{ needs.test-isolated.result }}",
+      "SHARED_RESULT: ${{ needs.test-shared.result }}",
+      "echo \"::error::test-isolated=${ISOLATED_RESULT}, test-shared=${SHARED_RESULT}\"",
+      "run: bun test src/shared/dist-bundle-bun-globals.test.ts",
+    ],
+  },
+  {
+    path: new URL("../.github/workflows/publish.yml", import.meta.url),
+    testRuns: ["run: bun run script/run-ci-tests.ts"],
+  },
 ]
 
 describe("test workflows", () => {
   test("use pure bun test for workflows", () => {
-    for (const workflowPath of workflowPaths) {
+    for (const workflowCheck of workflowChecks) {
       // #given
-      const workflow = readFileSync(workflowPath, "utf8")
+      const workflow = readFileSync(workflowCheck.path, "utf8")
 
-      expect(workflow).toContain("- name: Run tests")
-      expect(workflow).toMatch(/run: bun (test|run script\/run-ci-tests\.ts)/)
+      for (const testRun of workflowCheck.testRuns) {
+        expect(workflow).toContain(testRun)
+      }
     }
   })
 })

script/run-ci-tests.test.ts

@@ -1,4 +1,5 @@
 import { describe, expect, test } from "bun:test"
+import { selectCiTestTargets } from "./run-ci-tests"
 
 describe("test script isolation", () => {
   test("#given mock.module tests in the suite #then bun run test uses the isolated CI runner", async () => {
@@ -8,4 +9,37 @@ describe("test script isolation", () => {
     //#then
     expect(packageJson.scripts.test).toBe("bun run script/run-ci-tests.ts")
   })
+
+  test("#given isolated test shards #when selecting targets #then shards are deterministic and complete", () => {
+    // given
+    const ciTestPlan = {
+      isolatedModuleMockFiles: [],
+      isolatedTestTargets: ["a.test.ts", "b.test.ts", "c.test.ts", "d.test.ts", "e.test.ts"],
+      sharedTestFiles: ["shared.test.ts"],
+    }
+
+    // when
+    const shardOne = selectCiTestTargets(ciTestPlan, { phase: "isolated", shardCount: 2, shardIndex: 0 })
+    const shardTwo = selectCiTestTargets(ciTestPlan, { phase: "isolated", shardCount: 2, shardIndex: 1 })
+
+    // then
+    expect(shardOne).toEqual({ isolatedTestTargets: ["a.test.ts", "c.test.ts", "e.test.ts"], sharedTestFiles: [] })
+    expect(shardTwo).toEqual({ isolatedTestTargets: ["b.test.ts", "d.test.ts"], sharedTestFiles: [] })
+    expect([...shardOne.isolatedTestTargets, ...shardTwo.isolatedTestTargets].sort()).toEqual(ciTestPlan.isolatedTestTargets)
+  })
+
+  test("#given shared phase #when selecting targets #then only shared tests run", () => {
+    // given
+    const ciTestPlan = {
+      isolatedModuleMockFiles: [],
+      isolatedTestTargets: ["isolated.test.ts"],
+      sharedTestFiles: ["shared.test.ts"],
+    }
+
+    // when
+    const selectedTargets = selectCiTestTargets(ciTestPlan, { phase: "shared", shardCount: 1, shardIndex: 0 })
+
+    // then
+    expect(selectedTargets).toEqual({ isolatedTestTargets: [], sharedTestFiles: ["shared.test.ts"] })
+  })
 })

script/run-ci-tests.ts

@@ -6,6 +6,19 @@ type CiTestPlan = {
   sharedTestFiles: string[]
 }
 
+type CiTestPhase = "all" | "isolated" | "shared"
+
+type CiTestRunOptions = {
+  phase: CiTestPhase
+  shardCount: number
+  shardIndex: number
+}
+
+type CiTestTargetSelection = {
+  isolatedTestTargets: string[]
+  sharedTestFiles: string[]
+}
+
 const TEST_ROOTS = ["bin", "script", "src"] as const
 const MODULE_MOCK_PATTERN = "mock.module("
 const ALWAYS_ISOLATED_TEST_FILES = [
@@ -62,6 +75,86 @@ function collapseNestedTargets(isolatedTargets: string[]): string[] {
   })
 }
 
+function readFlagValue(args: string[], flagName: string): string | null {
+  const prefix = `${flagName}=`
+  const flag = args.find((arg) => arg.startsWith(prefix))
+
+  return flag?.slice(prefix.length) ?? null
+}
+
+function parsePhase(rawPhase: string | null): CiTestPhase {
+  if (rawPhase === null) {
+    return "all"
+  }
+
+  if (rawPhase === "all" || rawPhase === "isolated" || rawPhase === "shared") {
+    return rawPhase
+  }
+
+  throw new Error(`Invalid --phase value: ${rawPhase}. Expected all, isolated, or shared.`)
+}
+
+function parsePositiveIntegerFlag(args: string[], flagName: string, defaultValue: number): number {
+  const rawValue = readFlagValue(args, flagName)
+  if (rawValue === null) {
+    return defaultValue
+  }
+
+  const parsedValue = Number(rawValue)
+  if (!Number.isInteger(parsedValue) || parsedValue < 1) {
+    throw new Error(`Invalid ${flagName} value: ${rawValue}. Expected a positive integer.`)
+  }
+
+  return parsedValue
+}
+
+function parseNonNegativeIntegerFlag(args: string[], flagName: string, defaultValue: number): number {
+  const rawValue = readFlagValue(args, flagName)
+  if (rawValue === null) {
+    return defaultValue
+  }
+
+  const parsedValue = Number(rawValue)
+  if (!Number.isInteger(parsedValue) || parsedValue < 0) {
+    throw new Error(`Invalid ${flagName} value: ${rawValue}. Expected a non-negative integer.`)
+  }
+
+  return parsedValue
+}
+
+function parseCiTestRunOptions(args: string[]): CiTestRunOptions {
+  const phase = parsePhase(readFlagValue(args, "--phase"))
+  const shardCount = parsePositiveIntegerFlag(args, "--shard-count", 1)
+  const shardIndex = parseNonNegativeIntegerFlag(args, "--shard-index", 0)
+
+  if (shardIndex >= shardCount) {
+    throw new Error(`Invalid --shard-index value: ${shardIndex}. Expected a value less than --shard-count ${shardCount}.`)
+  }
+
+  if (shardCount > 1 && phase !== "isolated") {
+    throw new Error("Test sharding is only supported with --phase=isolated.")
+  }
+
+  return { phase, shardCount, shardIndex }
+}
+
+function selectShard(testTargets: string[], shardCount: number, shardIndex: number): string[] {
+  if (shardCount === 1) {
+    return testTargets
+  }
+
+  return testTargets.filter((_, index) => index % shardCount === shardIndex)
+}
+
+export function selectCiTestTargets(ciTestPlan: CiTestPlan, options: CiTestRunOptions): CiTestTargetSelection {
+  const isolatedTestTargets = options.phase === "shared"
+    ? []
+    : selectShard(ciTestPlan.isolatedTestTargets, options.shardCount, options.shardIndex)
+  const sharedTestFiles = options.phase === "isolated" ? [] : ciTestPlan.sharedTestFiles
+
+  return { isolatedTestTargets, sharedTestFiles }
+}
+
 export async function createCiTestPlan(rootDirectory: string = process.cwd()): Promise<CiTestPlan> {
   const allTestFiles = await collectTestFiles(rootDirectory)
   const isolatedModuleMockFiles: string[] = []
@@ -97,16 +190,15 @@ async function runBunTest(testFiles: string[], label: string): Promise<void> {
   }
 
   console.log(`::group::${label}`)
-  
-  // For directory paths, exclude _auc* directories which are separate isolated targets
-  const args = testFiles.map(tf => {
-    if (tf.includes('/') && !tf.endsWith('.test.ts')) {
-      // It's a directory path, add negation glob
-      return [tf, '!_auc-*/**/*.test.ts']
+
+  const args = testFiles.map((testFile) => {
+    if (testFile.includes("/") && !testFile.endsWith(".test.ts")) {
+      return [testFile, "!_auc-*/**/*.test.ts"]
     }
-    return tf
+
+    return testFile
   }).flat()
-  
+
   const command = ["bun", "test", ...args]
   const spawnedProcess = Bun.spawn(command, {
     cwd: process.cwd(),
@@ -123,17 +215,25 @@ async function runBunTest(testFiles: string[], label: string): Promise<void> {
 }
 
 async function main(): Promise<void> {
+  const options = parseCiTestRunOptions(process.argv.slice(2))
   const ciTestPlan = await createCiTestPlan()
+  const selectedTargets = selectCiTestTargets(ciTestPlan, options)
 
   console.log(
     `Detected ${ciTestPlan.isolatedModuleMockFiles.length} mock.module() test files, ${ciTestPlan.isolatedTestTargets.length} isolated targets, and ${ciTestPlan.sharedTestFiles.length} shared test files.`,
   )
 
-  for (const isolatedTestTarget of ciTestPlan.isolatedTestTargets) {
+  if (options.phase === "isolated" && options.shardCount > 1) {
+    console.log(
+      `Running isolated test shard ${options.shardIndex + 1}/${options.shardCount} with ${selectedTargets.isolatedTestTargets.length} targets.`,
+    )
+  }
+
+  for (const isolatedTestTarget of selectedTargets.isolatedTestTargets) {
     await runBunTest([isolatedTestTarget], `Isolated ${isolatedTestTarget}`)
   }
 
-  await runBunTest(ciTestPlan.sharedTestFiles, "Shared Bun test suite")
+  await runBunTest(selectedTargets.sharedTestFiles, "Shared Bun test suite")
 }
 
 export const moduleMockPattern = MODULE_MOCK_PATTERN