fix(team-mode): defer live mailbox acks
This commit is contained in:
@@ -17,18 +17,24 @@ export async function ackMessages(
|
|||||||
|
|
||||||
for (const messageId of messageIds) {
|
for (const messageId of messageIds) {
|
||||||
const messageFileName = `${messageId}.json`
|
const messageFileName = `${messageId}.json`
|
||||||
const sourcePath = path.join(inboxDir, messageFileName)
|
const sourcePaths = [
|
||||||
|
path.join(inboxDir, messageFileName),
|
||||||
|
path.join(inboxDir, `.delivering-${messageFileName}`),
|
||||||
|
]
|
||||||
const targetPath = path.join(processedDir, messageFileName)
|
const targetPath = path.join(processedDir, messageFileName)
|
||||||
|
|
||||||
try {
|
for (const sourcePath of sourcePaths) {
|
||||||
await rename(sourcePath, targetPath)
|
try {
|
||||||
} catch (error) {
|
await rename(sourcePath, targetPath)
|
||||||
const err = error as NodeJS.ErrnoException
|
break
|
||||||
if (err.code === "ENOENT") {
|
} catch (error) {
|
||||||
continue
|
const err = error as NodeJS.ErrnoException
|
||||||
}
|
if (err.code === "ENOENT") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
throw error
|
throw error
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -530,7 +530,7 @@ describe("createTeamSendMessageTool", () => {
|
|||||||
expect(message.from).toBe("m1")
|
expect(message.from).toBe("m1")
|
||||||
})
|
})
|
||||||
|
|
||||||
test("acks the message after live delivery so the transform hook does not redeliver", async () => {
|
test("keeps live-delivered messages reserved until the recipient idles", async () => {
|
||||||
// given
|
// given
|
||||||
const fixture = await createTeamFixture()
|
const fixture = await createTeamFixture()
|
||||||
const { client } = createRecordingClient()
|
const { client } = createRecordingClient()
|
||||||
@@ -546,9 +546,13 @@ describe("createTeamSendMessageTool", () => {
|
|||||||
// then
|
// then
|
||||||
const inboxDir = getInboxDir(resolveBaseDir(fixture.config), fixture.teamRunId, "m2")
|
const inboxDir = getInboxDir(resolveBaseDir(fixture.config), fixture.teamRunId, "m2")
|
||||||
const inboxEntries = (await readdir(inboxDir)).filter((entry) => entry.endsWith(".json"))
|
const inboxEntries = (await readdir(inboxDir)).filter((entry) => entry.endsWith(".json"))
|
||||||
const processedEntries = (await readdir(path.join(inboxDir, "processed"))).filter((entry) => entry.endsWith(".json"))
|
expect(inboxEntries).toHaveLength(1)
|
||||||
expect(inboxEntries).toHaveLength(0)
|
expect(inboxEntries[0]?.startsWith(".delivering-")).toBe(true)
|
||||||
expect(processedEntries).toHaveLength(1)
|
|
||||||
|
const { loadRuntimeState: loadState } = await import("../team-state-store/store")
|
||||||
|
const runtimeState = await loadState(fixture.teamRunId, fixture.config)
|
||||||
|
const recipient = runtimeState.members.find((member) => member.name === "m2")
|
||||||
|
expect(recipient?.pendingInjectedMessageIds).toHaveLength(1)
|
||||||
})
|
})
|
||||||
|
|
||||||
test("broadcast fans out live delivery to every member except the sender", async () => {
|
test("broadcast fans out live delivery to every member except the sender", async () => {
|
||||||
|
|||||||
@@ -1,22 +1,20 @@
|
|||||||
import { randomUUID } from "node:crypto"
|
import { randomUUID } from "node:crypto"
|
||||||
|
|
||||||
import { tool, type ToolDefinition } from "@opencode-ai/plugin/tool"
|
import { type ToolDefinition, tool } from "@opencode-ai/plugin/tool"
|
||||||
import { z } from "zod"
|
import { z } from "zod"
|
||||||
|
|
||||||
import type { TeamModeConfig } from "../../../config/schema/team-mode"
|
import type { TeamModeConfig } from "../../../config/schema/team-mode"
|
||||||
|
import { promptAsyncAfterSessionIdle } from "../../../hooks/shared/prompt-async-gate"
|
||||||
import { log } from "../../../shared/logger"
|
import { log } from "../../../shared/logger"
|
||||||
import { applyMemberSessionRouting, buildMemberPromptBody } from "../member-session-routing"
|
import { applyMemberSessionRouting, buildMemberPromptBody } from "../member-session-routing"
|
||||||
import { lookupTeamSession } from "../team-session-registry"
|
|
||||||
import { loadRuntimeState } from "../team-state-store/store"
|
|
||||||
import { buildEnvelope } from "../team-mailbox/poll"
|
import { buildEnvelope } from "../team-mailbox/poll"
|
||||||
import {
|
import {
|
||||||
commitDeliveryReservation,
|
|
||||||
releaseDeliveryReservation,
|
releaseDeliveryReservation,
|
||||||
reserveMessageForDelivery,
|
reserveMessageForDelivery,
|
||||||
} from "../team-mailbox/reservation"
|
} from "../team-mailbox/reservation"
|
||||||
import { BroadcastNotPermittedError, sendMessage } from "../team-mailbox/send"
|
import { BroadcastNotPermittedError, sendMessage } from "../team-mailbox/send"
|
||||||
import { promptAsyncAfterSessionIdle } from "../../../hooks/shared/prompt-async-gate"
|
import { lookupTeamSession } from "../team-session-registry"
|
||||||
|
import { loadRuntimeState, transitionRuntimeState } from "../team-state-store/store"
|
||||||
import type { Message } from "../types"
|
import type { Message } from "../types"
|
||||||
import { MessageSchema } from "../types"
|
import { MessageSchema } from "../types"
|
||||||
|
|
||||||
@@ -135,6 +133,25 @@ async function releaseReservationSafely(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function markLiveDeliveryPending(
|
||||||
|
teamRunId: string,
|
||||||
|
recipientName: string,
|
||||||
|
messageId: string,
|
||||||
|
config: TeamModeConfig,
|
||||||
|
): Promise<void> {
|
||||||
|
await transitionRuntimeState(teamRunId, (currentRuntimeState) => ({
|
||||||
|
...currentRuntimeState,
|
||||||
|
members: currentRuntimeState.members.map((member) => (
|
||||||
|
member.name === recipientName
|
||||||
|
? {
|
||||||
|
...member,
|
||||||
|
pendingInjectedMessageIds: Array.from(new Set([...member.pendingInjectedMessageIds, messageId])),
|
||||||
|
}
|
||||||
|
: member
|
||||||
|
)),
|
||||||
|
}), config)
|
||||||
|
}
|
||||||
|
|
||||||
async function deliverLive(
|
async function deliverLive(
|
||||||
client: LiveDeliveryClient,
|
client: LiveDeliveryClient,
|
||||||
message: Message,
|
message: Message,
|
||||||
@@ -207,8 +224,8 @@ async function deliverLive(
|
|||||||
})
|
})
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
await commitDeliveryReservation(reservation)
|
await markLiveDeliveryPending(teamRunId, recipientName, message.messageId, config)
|
||||||
log("[team-mailbox] live delivery committed", {
|
log("[team-mailbox] live delivery reserved until recipient idle", {
|
||||||
teamRunId,
|
teamRunId,
|
||||||
recipient: recipientName,
|
recipient: recipientName,
|
||||||
recipientSessionId,
|
recipientSessionId,
|
||||||
|
|||||||
@@ -80,6 +80,42 @@ function createRuntimeState(teamRunId: string, pendingInjectedMessageIds: string
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function createLeaderRuntimeState(teamRunId: string, pendingInjectedMessageIds: string[]): RuntimeState {
|
||||||
|
return {
|
||||||
|
version: 1,
|
||||||
|
teamRunId,
|
||||||
|
teamName: "team-alpha",
|
||||||
|
specSource: "project",
|
||||||
|
createdAt: 1,
|
||||||
|
status: "active",
|
||||||
|
leadSessionId: "lead-session",
|
||||||
|
members: [
|
||||||
|
{
|
||||||
|
name: "lead",
|
||||||
|
sessionId: "lead-session",
|
||||||
|
agentType: "leader",
|
||||||
|
status: "idle",
|
||||||
|
pendingInjectedMessageIds,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "worker",
|
||||||
|
sessionId: "member-session",
|
||||||
|
agentType: "general-purpose",
|
||||||
|
status: "idle",
|
||||||
|
pendingInjectedMessageIds: [],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
shutdownRequests: [],
|
||||||
|
bounds: {
|
||||||
|
maxMembers: 8,
|
||||||
|
maxParallelMembers: 4,
|
||||||
|
maxMessagesPerRun: 10000,
|
||||||
|
maxWallClockMinutes: 120,
|
||||||
|
maxMemberTurns: 500,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function seedRuntimeState(runtimeState: RuntimeState, config: TeamModeConfig): Promise<void> {
|
async function seedRuntimeState(runtimeState: RuntimeState, config: TeamModeConfig): Promise<void> {
|
||||||
await mkdir(path.join(config.base_dir ?? "", "runtime", runtimeState.teamRunId), { recursive: true })
|
await mkdir(path.join(config.base_dir ?? "", "runtime", runtimeState.teamRunId), { recursive: true })
|
||||||
await saveRuntimeState(runtimeState, config)
|
await saveRuntimeState(runtimeState, config)
|
||||||
@@ -103,6 +139,46 @@ async function seedUnreadMessage(
|
|||||||
}, teamRunId, config, { isLead: true, activeMembers: ["worker"] })
|
}, teamRunId, config, { isLead: true, activeMembers: ["worker"] })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function seedReservedUnreadMessage(
|
||||||
|
teamRunId: string,
|
||||||
|
config: TeamModeConfig,
|
||||||
|
messageId: string,
|
||||||
|
body: string,
|
||||||
|
timestamp: number,
|
||||||
|
): Promise<void> {
|
||||||
|
await sendMessage({
|
||||||
|
version: 1,
|
||||||
|
messageId,
|
||||||
|
from: "lead",
|
||||||
|
to: "worker",
|
||||||
|
kind: "message",
|
||||||
|
body,
|
||||||
|
timestamp,
|
||||||
|
}, teamRunId, config, {
|
||||||
|
isLead: true,
|
||||||
|
activeMembers: ["worker"],
|
||||||
|
reservedRecipients: new Set(["worker"]),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async function seedLeadUnreadMessage(
|
||||||
|
teamRunId: string,
|
||||||
|
config: TeamModeConfig,
|
||||||
|
messageId: string,
|
||||||
|
body: string,
|
||||||
|
timestamp: number,
|
||||||
|
): Promise<void> {
|
||||||
|
await sendMessage({
|
||||||
|
version: 1,
|
||||||
|
messageId,
|
||||||
|
from: "worker",
|
||||||
|
to: "lead",
|
||||||
|
kind: "message",
|
||||||
|
body,
|
||||||
|
timestamp,
|
||||||
|
}, teamRunId, config, { isLead: false, activeMembers: ["lead"] })
|
||||||
|
}
|
||||||
|
|
||||||
afterEach(async () => {
|
afterEach(async () => {
|
||||||
clearTeamSessionRegistry()
|
clearTeamSessionRegistry()
|
||||||
SessionCategoryRegistry.clear()
|
SessionCategoryRegistry.clear()
|
||||||
@@ -414,6 +490,81 @@ describe("createTeamIdleWakeHint", () => {
|
|||||||
expect(processedEntries.sort()).toEqual(messageIds.map((messageId) => `${messageId}.json`).sort())
|
expect(processedEntries.sort()).toEqual(messageIds.map((messageId) => `${messageId}.json`).sort())
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test("acks pending reserved live-delivery messages on idle", async () => {
|
||||||
|
// given
|
||||||
|
const baseDir = await createTemporaryBaseDir()
|
||||||
|
const config = createConfig(baseDir)
|
||||||
|
const teamRunId = randomUUID()
|
||||||
|
const messageId = randomUUID()
|
||||||
|
await seedRuntimeState(createRuntimeState(teamRunId, [messageId]), config)
|
||||||
|
await seedReservedUnreadMessage(teamRunId, config, messageId, "live delivery body", 100)
|
||||||
|
|
||||||
|
const handler = createTeamIdleWakeHint({
|
||||||
|
directory: "/tmp/project",
|
||||||
|
client: { session: { promptAsync: mock(async (_input: WakeHintPromptInput) => ({})) } },
|
||||||
|
}, config)
|
||||||
|
|
||||||
|
// when
|
||||||
|
await handler({
|
||||||
|
event: {
|
||||||
|
type: "session.idle",
|
||||||
|
properties: { sessionID: "member-session" },
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// then
|
||||||
|
const runtimeState = await loadRuntimeState(teamRunId, config)
|
||||||
|
expect(runtimeState.members[0]?.pendingInjectedMessageIds).toEqual([])
|
||||||
|
|
||||||
|
const inboxDir = getInboxDir(resolveBaseDir(config), teamRunId, "worker")
|
||||||
|
const inboxEntries = await readdir(inboxDir)
|
||||||
|
expect(inboxEntries).not.toContain(`.delivering-${messageId}.json`)
|
||||||
|
|
||||||
|
const processedEntries = await readdir(path.join(inboxDir, "processed"))
|
||||||
|
expect(processedEntries).toContain(`${messageId}.json`)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("acks pending lead messages on idle without sending a wake hint", async () => {
|
||||||
|
// given
|
||||||
|
const baseDir = await createTemporaryBaseDir()
|
||||||
|
const config = createConfig(baseDir)
|
||||||
|
const teamRunId = randomUUID()
|
||||||
|
const messageIds = [randomUUID(), randomUUID()]
|
||||||
|
await seedRuntimeState(createLeaderRuntimeState(teamRunId, messageIds), config)
|
||||||
|
await seedLeadUnreadMessage(teamRunId, config, messageIds[0], "one", 100)
|
||||||
|
await seedLeadUnreadMessage(teamRunId, config, messageIds[1], "two", 200)
|
||||||
|
|
||||||
|
const ackSpy = spyOn(ackModule, "ackMessages")
|
||||||
|
const promptAsyncSpy = mock(async (_input: WakeHintPromptInput) => ({}))
|
||||||
|
const handler = createTeamIdleWakeHint({
|
||||||
|
directory: "/tmp/project",
|
||||||
|
client: { session: { promptAsync: promptAsyncSpy } },
|
||||||
|
}, config)
|
||||||
|
|
||||||
|
// when
|
||||||
|
await handler({
|
||||||
|
event: {
|
||||||
|
type: "session.idle",
|
||||||
|
properties: { sessionID: "lead-session" },
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// then
|
||||||
|
expect(ackSpy).toHaveBeenCalledTimes(1)
|
||||||
|
expect(ackSpy).toHaveBeenCalledWith(teamRunId, "lead", messageIds, config)
|
||||||
|
expect(promptAsyncSpy).not.toHaveBeenCalled()
|
||||||
|
|
||||||
|
const runtimeState = await loadRuntimeState(teamRunId, config)
|
||||||
|
expect(runtimeState.members[0]?.pendingInjectedMessageIds).toEqual([])
|
||||||
|
|
||||||
|
const inboxDir = getInboxDir(resolveBaseDir(config), teamRunId, "lead")
|
||||||
|
const inboxEntries = await readdir(inboxDir)
|
||||||
|
expect(inboxEntries).toContain("processed")
|
||||||
|
|
||||||
|
const processedEntries = await readdir(path.join(inboxDir, "processed"))
|
||||||
|
expect(processedEntries.sort()).toEqual(messageIds.map((messageId) => `${messageId}.json`).sort())
|
||||||
|
})
|
||||||
|
|
||||||
test("sends a wake hint during the spawn race when the registry tracks the fresh member session before disk state persists it", async () => {
|
test("sends a wake hint during the spawn race when the registry tracks the fresh member session before disk state persists it", async () => {
|
||||||
// given
|
// given
|
||||||
const baseDir = await createTemporaryBaseDir()
|
const baseDir = await createTemporaryBaseDir()
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
import type { TeamModeConfig } from "../../config/schema/team-mode"
|
import type { TeamModeConfig } from "../../config/schema/team-mode"
|
||||||
import { ackMessages } from "../../features/team-mode/team-mailbox/ack"
|
|
||||||
import { listUnreadMessages } from "../../features/team-mode/team-mailbox/inbox"
|
|
||||||
import { loadRuntimeState, listActiveTeams, transitionRuntimeState } from "../../features/team-mode/team-state-store/store"
|
|
||||||
import { findResolvedMemberSession } from "../../features/team-mode/member-session-resolution"
|
import { findResolvedMemberSession } from "../../features/team-mode/member-session-resolution"
|
||||||
import {
|
import {
|
||||||
applyMemberSessionRouting,
|
applyMemberSessionRouting,
|
||||||
buildMemberPromptBody,
|
buildMemberPromptBody,
|
||||||
} from "../../features/team-mode/member-session-routing"
|
} from "../../features/team-mode/member-session-routing"
|
||||||
|
import { ackMessages } from "../../features/team-mode/team-mailbox/ack"
|
||||||
|
import { listUnreadMessages } from "../../features/team-mode/team-mailbox/inbox"
|
||||||
|
import { loadRuntimeState, transitionRuntimeState } from "../../features/team-mode/team-state-store/store"
|
||||||
import { resolveSessionEventID } from "../../shared/event-session-id"
|
import { resolveSessionEventID } from "../../shared/event-session-id"
|
||||||
import { log } from "../../shared/logger"
|
import { log } from "../../shared/logger"
|
||||||
import { promptAsyncAfterSessionIdle } from "../shared/prompt-async-gate"
|
import { promptAsyncAfterSessionIdle } from "../shared/prompt-async-gate"
|
||||||
@@ -59,7 +59,7 @@ export function createTeamIdleWakeHint(ctx: TeamIdleWakeHintContext, config: Tea
|
|||||||
|
|
||||||
const runtimeState = await loadRuntimeState(runtimeMember.teamRunId, config)
|
const runtimeState = await loadRuntimeState(runtimeMember.teamRunId, config)
|
||||||
const memberEntry = runtimeState.members.find((member) => member.name === runtimeMember.memberName)
|
const memberEntry = runtimeState.members.find((member) => member.name === runtimeMember.memberName)
|
||||||
if (!memberEntry || memberEntry.agentType === "leader") {
|
if (!memberEntry) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -88,6 +88,17 @@ export function createTeamIdleWakeHint(ctx: TeamIdleWakeHintContext, config: Tea
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (memberEntry.agentType === "leader") {
|
||||||
|
log("team lead idle handled without wake hint", {
|
||||||
|
event: "team-mode-lead-idle-ack-only",
|
||||||
|
teamRunId: runtimeState.teamRunId,
|
||||||
|
memberName: memberEntry.name,
|
||||||
|
sessionID,
|
||||||
|
ackedCount: pendingInjectedMessageIds.length,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if (typeof ctx.client.session.promptAsync !== "function") {
|
if (typeof ctx.client.session.promptAsync !== "function") {
|
||||||
log("team idle wake hint skipped without promptAsync", {
|
log("team idle wake hint skipped without promptAsync", {
|
||||||
event: "team-mode-idle-wake-hint-skipped",
|
event: "team-mode-idle-wake-hint-skipped",
|
||||||
|
|||||||
@@ -2,12 +2,14 @@
|
|||||||
|
|
||||||
import { afterEach, describe, expect, test } from "bun:test"
|
import { afterEach, describe, expect, test } from "bun:test"
|
||||||
import { randomUUID } from "node:crypto"
|
import { randomUUID } from "node:crypto"
|
||||||
import { mkdtemp, mkdir, rm } from "node:fs/promises"
|
import { mkdtemp, mkdir, readdir, rm } from "node:fs/promises"
|
||||||
import { tmpdir } from "node:os"
|
import { tmpdir } from "node:os"
|
||||||
import path from "node:path"
|
import path from "node:path"
|
||||||
|
|
||||||
import { TeamModeConfigSchema } from "../../config/schema/team-mode"
|
import { TeamModeConfigSchema } from "../../config/schema/team-mode"
|
||||||
import type { TeamModeConfig } from "../../config/schema/team-mode"
|
import type { TeamModeConfig } from "../../config/schema/team-mode"
|
||||||
|
import { sendMessage } from "../../features/team-mode/team-mailbox/send"
|
||||||
|
import { getInboxDir, resolveBaseDir } from "../../features/team-mode/team-registry/paths"
|
||||||
import {
|
import {
|
||||||
clearTeamSessionRegistry,
|
clearTeamSessionRegistry,
|
||||||
registerTeamSession,
|
registerTeamSession,
|
||||||
@@ -57,11 +59,37 @@ function createRuntimeState(teamRunId: string): RuntimeState {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function createRuntimeStateWithPendingMessage(teamRunId: string, messageId: string): RuntimeState {
|
||||||
|
const runtimeState = createRuntimeState(teamRunId)
|
||||||
|
const worker = runtimeState.members[0]
|
||||||
|
if (worker === undefined) {
|
||||||
|
throw new Error("worker member missing from fixture")
|
||||||
|
}
|
||||||
|
worker.pendingInjectedMessageIds = [messageId]
|
||||||
|
return runtimeState
|
||||||
|
}
|
||||||
|
|
||||||
async function seedRuntimeState(runtimeState: RuntimeState, config: TeamModeConfig): Promise<void> {
|
async function seedRuntimeState(runtimeState: RuntimeState, config: TeamModeConfig): Promise<void> {
|
||||||
await mkdir(path.join(config.base_dir ?? "", "runtime", runtimeState.teamRunId), { recursive: true })
|
await mkdir(path.join(config.base_dir ?? "", "runtime", runtimeState.teamRunId), { recursive: true })
|
||||||
await saveRuntimeState(runtimeState, config)
|
await saveRuntimeState(runtimeState, config)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function seedReservedMessage(teamRunId: string, config: TeamModeConfig, messageId: string): Promise<void> {
|
||||||
|
await sendMessage({
|
||||||
|
version: 1,
|
||||||
|
messageId,
|
||||||
|
from: "lead",
|
||||||
|
to: "worker",
|
||||||
|
kind: "message",
|
||||||
|
body: "pending live delivery",
|
||||||
|
timestamp: 1,
|
||||||
|
}, teamRunId, config, {
|
||||||
|
isLead: true,
|
||||||
|
activeMembers: ["worker"],
|
||||||
|
reservedRecipients: new Set(["worker"]),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
afterEach(async () => {
|
afterEach(async () => {
|
||||||
clearTeamSessionRegistry()
|
clearTeamSessionRegistry()
|
||||||
await Promise.all(temporaryDirectories.splice(0).map(async (directoryPath) => {
|
await Promise.all(temporaryDirectories.splice(0).map(async (directoryPath) => {
|
||||||
@@ -169,4 +197,33 @@ describe("createTeamMemberErrorHandler", () => {
|
|||||||
expect(correctRuntimeState.members[0]?.status).toBe("errored")
|
expect(correctRuntimeState.members[0]?.status).toBe("errored")
|
||||||
expect(wrongRuntimeState.members[0]?.status).toBe("running")
|
expect(wrongRuntimeState.members[0]?.status).toBe("running")
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test("requeues pending live-delivery messages when the recipient session errors before idle ack", async () => {
|
||||||
|
// given
|
||||||
|
const baseDir = await createTemporaryBaseDir()
|
||||||
|
const config = createConfig(baseDir)
|
||||||
|
const teamRunId = randomUUID()
|
||||||
|
const messageId = randomUUID()
|
||||||
|
await seedRuntimeState(createRuntimeStateWithPendingMessage(teamRunId, messageId), config)
|
||||||
|
await seedReservedMessage(teamRunId, config, messageId)
|
||||||
|
const handler = createTeamMemberErrorHandler(config)
|
||||||
|
|
||||||
|
// when
|
||||||
|
await handler({
|
||||||
|
event: {
|
||||||
|
type: "session.error",
|
||||||
|
properties: { sessionID: "member-session", error: new Error("late prompt failure") },
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// then
|
||||||
|
const runtimeState = await loadRuntimeState(teamRunId, config)
|
||||||
|
expect(runtimeState.members[0]?.status).toBe("errored")
|
||||||
|
expect(runtimeState.members[0]?.pendingInjectedMessageIds).toEqual([])
|
||||||
|
|
||||||
|
const inboxEntries = await readdir(getInboxDir(resolveBaseDir(config), teamRunId, "worker"))
|
||||||
|
expect(inboxEntries).toContain(`${messageId}.json`)
|
||||||
|
expect(inboxEntries).not.toContain(`.delivering-${messageId}.json`)
|
||||||
|
expect(inboxEntries).not.toContain("processed")
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -1,5 +1,9 @@
|
|||||||
import type { TeamModeConfig } from "../../config/schema/team-mode"
|
import type { TeamModeConfig } from "../../config/schema/team-mode"
|
||||||
import { findResolvedMemberSession } from "../../features/team-mode/member-session-resolution"
|
import { findResolvedMemberSession } from "../../features/team-mode/member-session-resolution"
|
||||||
|
import {
|
||||||
|
releaseDeliveryReservation,
|
||||||
|
reserveMessageForDelivery,
|
||||||
|
} from "../../features/team-mode/team-mailbox/reservation"
|
||||||
import { loadRuntimeState, transitionRuntimeState } from "../../features/team-mode/team-state-store/store"
|
import { loadRuntimeState, transitionRuntimeState } from "../../features/team-mode/team-state-store/store"
|
||||||
import { resolveSessionEventID } from "../../shared/event-session-id"
|
import { resolveSessionEventID } from "../../shared/event-session-id"
|
||||||
import { log } from "../../shared/logger"
|
import { log } from "../../shared/logger"
|
||||||
@@ -11,6 +15,22 @@ function getErroredSessionID(properties: unknown): string | undefined {
|
|||||||
return resolveSessionEventID(properties)
|
return resolveSessionEventID(properties)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function requeuePendingLiveDeliveries(
|
||||||
|
teamRunId: string,
|
||||||
|
memberName: string,
|
||||||
|
messageIds: readonly string[],
|
||||||
|
config: TeamModeConfig,
|
||||||
|
): Promise<void> {
|
||||||
|
for (const messageId of messageIds) {
|
||||||
|
const reservation = await reserveMessageForDelivery(teamRunId, memberName, messageId, config)
|
||||||
|
if (reservation === null) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
await releaseDeliveryReservation(reservation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export function createTeamMemberErrorHandler(config: TeamModeConfig): HookImpl {
|
export function createTeamMemberErrorHandler(config: TeamModeConfig): HookImpl {
|
||||||
return async ({ event }: HookInput): Promise<void> => {
|
return async ({ event }: HookInput): Promise<void> => {
|
||||||
if (event.type !== "session.error") return
|
if (event.type !== "session.error") return
|
||||||
@@ -25,11 +45,19 @@ export function createTeamMemberErrorHandler(config: TeamModeConfig): HookImpl {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const runtimeState = await loadRuntimeState(runtimeMember.teamRunId, config)
|
const runtimeState = await loadRuntimeState(runtimeMember.teamRunId, config)
|
||||||
|
const memberEntry = runtimeState.members.find((member) => member.name === runtimeMember.memberName)
|
||||||
|
const pendingInjectedMessageIds = memberEntry?.pendingInjectedMessageIds ?? []
|
||||||
|
await requeuePendingLiveDeliveries(
|
||||||
|
runtimeState.teamRunId,
|
||||||
|
runtimeMember.memberName,
|
||||||
|
pendingInjectedMessageIds,
|
||||||
|
config,
|
||||||
|
)
|
||||||
await transitionRuntimeState(runtimeState.teamRunId, (currentRuntimeState) => ({
|
await transitionRuntimeState(runtimeState.teamRunId, (currentRuntimeState) => ({
|
||||||
...currentRuntimeState,
|
...currentRuntimeState,
|
||||||
members: currentRuntimeState.members.map((member) => (
|
members: currentRuntimeState.members.map((member) => (
|
||||||
member.name === runtimeMember.memberName
|
member.name === runtimeMember.memberName
|
||||||
? { ...member, status: "errored" }
|
? { ...member, status: "errored", pendingInjectedMessageIds: [] }
|
||||||
: member
|
: member
|
||||||
)),
|
)),
|
||||||
}), config)
|
}), config)
|
||||||
|
|||||||
Reference in New Issue
Block a user