fix(team-mode): close peer message delivery races

2026-05-20 11:42:38 +09:00
parent d3e218f912
commit 6df148f0f6
6 changed files with 426 additions and 46 deletions
@@ -26,6 +26,7 @@ import {
  releaseAllPromptAsyncReservationsForTesting,
  releasePromptAsyncReservation,
 } from "../shared/prompt-async-gate"
+import { createTeamMemberErrorHandler } from "./team-member-error-handler"
 import { createTeamIdleWakeHint } from "./team-idle-wake-hint"

 type WakeHintPromptInput = {
@@ -609,6 +610,54 @@ describe("createTeamIdleWakeHint", () => {
    expect(inboxEntries).not.toContain(`${messageId}.json`)
  })

+  test("#given pending live delivery is requeued by session.error #when stale idle handler continues #then it does not ack the requeued message", async () => {
+    // given
+    const baseDir = await createTemporaryBaseDir()
+    const config = createConfig(baseDir)
+    const teamRunId = randomUUID()
+    const messageId = randomUUID()
+    await seedRuntimeState(createRuntimeState(teamRunId, [messageId]), config)
+    await seedReservedUnreadMessage(teamRunId, config, messageId, "live delivery body", 100)
+    const errorHandler = createTeamMemberErrorHandler(config)
+
+    const handler = createTeamIdleWakeHint({
+      directory: "/tmp/project",
+      client: {
+        session: {
+          promptAsync: mock(async (_input: WakeHintPromptInput) => ({})),
+          status: async () => {
+            await errorHandler({
+              event: {
+                type: "session.error",
+                properties: { sessionID: "member-session", error: new Error("late prompt failure") },
+              },
+            })
+            return { data: { "member-session": { type: "idle" } } }
+          },
+        },
+      },
+    }, config, { idleSettleMs: 0 })
+
+    // when
+    await handler({
+      event: {
+        type: "session.idle",
+        properties: { sessionID: "member-session" },
+      },
+    })
+
+    // then
+    const runtimeState = await loadRuntimeState(teamRunId, config)
+    expect(runtimeState.members[0]?.status).toBe("errored")
+    expect(runtimeState.members[0]?.pendingInjectedMessageIds).toEqual([])
+
+    const inboxDir = getInboxDir(resolveBaseDir(config), teamRunId, "worker")
+    const inboxEntries = await readdir(inboxDir)
+    expect(inboxEntries).toContain(`${messageId}.json`)
+    expect(inboxEntries).not.toContain(`.delivering-${messageId}.json`)
+    expect(inboxEntries).not.toContain("processed")
+  })
+
  test("#given a pending live-delivery ack and later unread message #when member idles after the live reply #then it wakes the member for the unread message", async () => {
    // given
    const baseDir = await createTemporaryBaseDir()
@@ -8,7 +8,7 @@ import { ackMessages } from "../../features/team-mode/team-mailbox/ack"
 import { listUnreadMessages } from "../../features/team-mode/team-mailbox/inbox"
 import { loadRuntimeState, transitionRuntimeState } from "../../features/team-mode/team-state-store/store"
 import { resolveSessionEventID } from "../../shared/event-session-id"
-import { isAmbiguousPromptDispatchFailure } from "../../shared/prompt-failure-classifier"
+import { isAmbiguousPostDispatchPromptFailure } from "../../shared/prompt-failure-classifier"
 import { log } from "../../shared/logger"
 import { isSessionActive, settleAfterSessionIdle } from "../../shared/session-idle-settle"
 import { dispatchInternalPrompt, isInternalPromptDispatchAccepted } from "../shared/prompt-async-gate"
@@ -51,6 +51,45 @@ function buildWakeHintBatchKey(teamRunId: string, memberName: string, messageIds
  return `${teamRunId}:${memberName}:${messageIds.toSorted().join(",")}`
 }

+async function claimPendingMessageAcks(
+  teamRunId: string,
+  memberName: string,
+  messageIds: readonly string[],
+  config: TeamModeConfig,
+): Promise<string[]> {
+  if (messageIds.length === 0) return []
+
+  let claimedMessageIds: string[] = []
+  const candidateMessageIds = new Set(messageIds)
+  await transitionRuntimeState(teamRunId, (currentRuntimeState) => {
+    const currentMember = currentRuntimeState.members.find((member) => member.name === memberName)
+    if (currentMember === undefined) {
+      claimedMessageIds = []
+      return currentRuntimeState
+    }
+
+    claimedMessageIds = currentMember.pendingInjectedMessageIds.filter((messageId) => candidateMessageIds.has(messageId))
+    if (claimedMessageIds.length === 0) {
+      return currentRuntimeState
+    }
+
+    const claimedMessageIdSet = new Set(claimedMessageIds)
+    return {
+      ...currentRuntimeState,
+      members: currentRuntimeState.members.map((member) => (
+        member.name === memberName
+          ? {
+            ...member,
+            pendingInjectedMessageIds: member.pendingInjectedMessageIds.filter((messageId) => !claimedMessageIdSet.has(messageId)),
+          }
+          : member
+      )),
+    }
+  }, config)
+
+  return claimedMessageIds
+}
+
 export function createTeamIdleWakeHint(ctx: TeamIdleWakeHintContext, config: TeamModeConfig, options?: TeamIdleWakeHintOptions): HookImpl {
  const recentWakeHintBatches = new Map<string, number>()

@@ -88,41 +127,61 @@ export function createTeamIdleWakeHint(ctx: TeamIdleWakeHintContext, config: Tea
          }
        }

-        await ackMessages(runtimeState.teamRunId, memberEntry.name, pendingInjectedMessageIds, config)
-        await transitionRuntimeState(runtimeState.teamRunId, (currentRuntimeState) => ({
-          ...currentRuntimeState,
-          members: currentRuntimeState.members.map((member) => (
-            member.name === memberEntry.name
-              ? { ...member, pendingInjectedMessageIds: [] }
-              : member
-          )),
-        }), config)
-        log("team idle handled pending live delivery ack", {
-          event: "team-mode-idle-pending-ack",
-          teamRunId: runtimeState.teamRunId,
-          memberName: memberEntry.name,
-          sessionID,
-          ackedCount: pendingInjectedMessageIds.length,
-        })
+        const claimedMessageIds = await claimPendingMessageAcks(
+          runtimeState.teamRunId,
+          memberEntry.name,
+          pendingInjectedMessageIds,
+          config,
+        )
+        if (claimedMessageIds.length > 0) {
+          await ackMessages(runtimeState.teamRunId, memberEntry.name, claimedMessageIds, config)
+          log("team idle handled pending live delivery ack", {
+            event: "team-mode-idle-pending-ack",
+            teamRunId: runtimeState.teamRunId,
+            memberName: memberEntry.name,
+            sessionID,
+            ackedCount: claimedMessageIds.length,
+          })
+        }
      }

-      const unreadMessages = await listUnreadMessages(runtimeState.teamRunId, memberEntry.name, config)
+      const latestRuntimeState = await loadRuntimeState(runtimeMember.teamRunId, config)
+      const latestMemberEntry = latestRuntimeState.members.find((member) => member.name === runtimeMember.memberName)
+      if (!latestMemberEntry) {
+        return
+      }
+      if (
+        latestMemberEntry.status === "errored"
+        || latestMemberEntry.status === "completed"
+        || latestMemberEntry.status === "shutdown_approved"
+      ) {
+        log("team idle wake hint skipped because member is no longer idle", {
+          event: "team-mode-idle-member-not-idle",
+          teamRunId: latestRuntimeState.teamRunId,
+          memberName: latestMemberEntry.name,
+          sessionID,
+          status: latestMemberEntry.status,
+        })
+        return
+      }
+
+      const unreadMessages = await listUnreadMessages(latestRuntimeState.teamRunId, latestMemberEntry.name, config)
      if (unreadMessages.length === 0) {
        log("team idle handled without wake hint", {
          event: "team-mode-idle-ack-only",
-          teamRunId: runtimeState.teamRunId,
-          memberName: memberEntry.name,
+          teamRunId: latestRuntimeState.teamRunId,
+          memberName: latestMemberEntry.name,
          sessionID,
          ackedCount: pendingInjectedMessageIds.length,
        })
        return
      }

-      if (memberEntry.agentType === "leader") {
+      if (latestMemberEntry.agentType === "leader") {
        log("team lead idle handled without wake hint", {
          event: "team-mode-lead-idle-ack-only",
-          teamRunId: runtimeState.teamRunId,
-          memberName: memberEntry.name,
+          teamRunId: latestRuntimeState.teamRunId,
+          memberName: latestMemberEntry.name,
          sessionID,
          ackedCount: pendingInjectedMessageIds.length,
        })
@@ -132,8 +191,8 @@ export function createTeamIdleWakeHint(ctx: TeamIdleWakeHintContext, config: Tea
      if (typeof ctx.client.session.promptAsync !== "function") {
        log("team idle wake hint skipped without promptAsync", {
          event: "team-mode-idle-wake-hint-skipped",
-          teamRunId: runtimeState.teamRunId,
-          memberName: memberEntry.name,
+          teamRunId: latestRuntimeState.teamRunId,
+          memberName: latestMemberEntry.name,
          sessionID,
          unreadCount: unreadMessages.length,
        })
@@ -142,16 +201,16 @@ export function createTeamIdleWakeHint(ctx: TeamIdleWakeHintContext, config: Tea

      const now = Date.now()
      const wakeHintBatchKey = buildWakeHintBatchKey(
-        runtimeState.teamRunId,
-        memberEntry.name,
+        latestRuntimeState.teamRunId,
+        latestMemberEntry.name,
        unreadMessages.map((message) => message.messageId),
      )
      const suppressedUntil = recentWakeHintBatches.get(wakeHintBatchKey)
      if (suppressedUntil !== undefined && suppressedUntil > now) {
        log("team idle wake hint skipped for recently hinted unread batch", {
          event: "team-mode-idle-wake-hint-duplicate-suppressed",
-          teamRunId: runtimeState.teamRunId,
-          memberName: memberEntry.name,
+          teamRunId: latestRuntimeState.teamRunId,
+          memberName: latestMemberEntry.name,
          sessionID,
          unreadCount: unreadMessages.length,
        })
@@ -161,7 +220,7 @@ export function createTeamIdleWakeHint(ctx: TeamIdleWakeHintContext, config: Tea
        recentWakeHintBatches.delete(wakeHintBatchKey)
      }

-      applyMemberSessionRouting(sessionID, memberEntry)
+      applyMemberSessionRouting(sessionID, latestMemberEntry)
      const promptResult = await dispatchInternalPrompt({
        mode: "async",
        client: ctx.client,
@@ -171,18 +230,18 @@ export function createTeamIdleWakeHint(ctx: TeamIdleWakeHintContext, config: Tea
        queueBehavior: "defer",
        input: {
          path: { id: sessionID },
-          body: buildMemberPromptBody(memberEntry, buildWakeHint(unreadMessages.length)),
+          body: buildMemberPromptBody(latestMemberEntry, buildWakeHint(unreadMessages.length)),
          query: { directory: ctx.directory },
        },
      })
      if (!isInternalPromptDispatchAccepted(promptResult)) {
-        if (promptResult.status === "failed" && isAmbiguousPromptDispatchFailure(promptResult.error)) {
+        if (promptResult.status === "failed" && isAmbiguousPostDispatchPromptFailure(promptResult)) {
          recentWakeHintBatches.set(wakeHintBatchKey, Date.now() + WAKE_HINT_DUPLICATE_SUPPRESSION_MS)
        }
        log("team idle wake hint skipped by promptAsync gate", {
          event: "team-mode-idle-wake-hint-gated",
-          teamRunId: runtimeState.teamRunId,
-          memberName: memberEntry.name,
+          teamRunId: latestRuntimeState.teamRunId,
+          memberName: latestMemberEntry.name,
          sessionID,
          unreadCount: unreadMessages.length,
          status: promptResult.status,
@@ -193,8 +252,8 @@ export function createTeamIdleWakeHint(ctx: TeamIdleWakeHintContext, config: Tea

      log("team idle wake hint sent", {
        event: "team-mode-idle-wake-hint",
-        teamRunId: runtimeState.teamRunId,
-        memberName: memberEntry.name,
+        teamRunId: latestRuntimeState.teamRunId,
+        memberName: latestMemberEntry.name,
        sessionID,
        unreadCount: unreadMessages.length,
        ackedCount: pendingInjectedMessageIds.length,