diff --git a/server/src/services/agent-invokability.ts b/server/src/services/agent-invokability.ts index 75726c92..b9b39d59 100644 --- a/server/src/services/agent-invokability.ts +++ b/server/src/services/agent-invokability.ts @@ -32,7 +32,7 @@ export type AgentInvokability = invalidOrgChain: boolean; }; -const DIRECT_NON_INVOKABLE_STATUSES = new Set([ +export const DIRECT_NON_INVOKABLE_STATUSES = new Set([ "paused", "terminated", "pending_approval", diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 66e0d612..dc089b71 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -163,6 +163,7 @@ import { evaluateAgentInvokability, evaluateAgentInvokabilityFromDb, shouldCancelRunsForNonInvokableAgent, + DIRECT_NON_INVOKABLE_STATUSES, type AgentOrgRow, } from "./agent-invokability.js"; import { @@ -8926,25 +8927,51 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .then((rows) => rows[0] ?? null); if (runningWithSession) run = runningWithSession; + // Pause Durability: flip to "running" ONLY if the agent is still invokable. + // Atomic conditional UPDATE is the sole gate (no read-then-write); 0 rows => abort. const runningAgent = await db .update(agents) .set({ status: "running", updatedAt: new Date() }) - .where(eq(agents.id, agent.id)) + .where(and(eq(agents.id, agent.id), notInArray(agents.status, [...DIRECT_NON_INVOKABLE_STATUSES]))) .returning() .then((rows) => rows[0] ?? null); - if (runningAgent) { - publishLiveEvent({ - companyId: runningAgent.companyId, - type: "agent.status", - payload: { - agentId: runningAgent.id, - status: runningAgent.status, - outcome: "running", - }, + if (!runningAgent) { + logger.warn( + { agentId: agent.id, runId: run.id, previousStatus: agent.status }, + "execution-start aborted: agent not invokable", + ); + const abortReason = "Cancelled: agent not invokable at execution-start"; + await setRunStatus(run.id, "cancelled", { + finishedAt: new Date(), + error: abortReason, + errorCode: "agent_not_invokable", + ...(agent ? { + resultJson: mergeRunStopMetadataForAgent(agent, "cancelled", { + resultJson: parseObject(run.resultJson), + errorCode: "agent_not_invokable", + errorMessage: abortReason, + }), + } : {}), }); + await setWakeupStatus(run.wakeupRequestId, "cancelled", { + finishedAt: new Date(), + error: abortReason, + }); + await releaseIssueExecutionAndPromote(run); + return; } + publishLiveEvent({ + companyId: runningAgent.companyId, + type: "agent.status", + payload: { + agentId: runningAgent.id, + status: runningAgent.status, + outcome: "running", + }, + }); + const currentRun = run; await appendRunEvent(currentRun, seq++, { eventType: "lifecycle", @@ -11317,7 +11344,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return cancelled; } - async function cancelActiveForAgentInternal(agentId: string, reason = "Cancelled due to agent pause") { + async function cancelActiveForAgentInternal(agentId: string, reason = "Cancelled due to agent pause", errorCode = "cancelled") { const agent = await getAgent(agentId); const runs = await db .select() @@ -11328,11 +11355,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) await setRunStatus(run.id, "cancelled", { finishedAt: new Date(), error: reason, - errorCode: "cancelled", + errorCode, ...(agent ? { resultJson: mergeRunStopMetadataForAgent(agent, "cancelled", { resultJson: parseObject(run.resultJson), - errorCode: "cancelled", + errorCode, errorMessage: reason, }), } : {}), @@ -11755,7 +11782,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) cancelRun: (runId: string, reason?: string, options?: CancelRunOptions) => cancelRunInternal(runId, reason, options), - cancelActiveForAgent: (agentId: string, reason?: string) => cancelActiveForAgentInternal(agentId, reason), + /** + * Pause-only. Emits errorCode "agent_paused" unconditionally; its sole caller is the + * agent pause route. For non-pause cancellations use cancelRun, or call the internal + * cancelActiveForAgentInternal(agentId, reason, errorCode) with an explicit errorCode. + */ + cancelActiveForAgent: (agentId: string, reason?: string) => cancelActiveForAgentInternal(agentId, reason, "agent_paused"), cancelInvocationsForAgents: (agentIds: string[], reason: string) => cancelInvocationsForAgentsInternal(agentIds, reason), diff --git a/server/src/services/recovery/service.pause-durability.test.ts b/server/src/services/recovery/service.pause-durability.test.ts new file mode 100644 index 00000000..5e63043f --- /dev/null +++ b/server/src/services/recovery/service.pause-durability.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { classifyContinuationFailure } from "./service.js"; + +const run = (errorCode: string | null) => + ({ errorCode } as unknown as Parameters[0]); + +describe("pause durability: continuation retry classification", () => { + it("agent_paused is retryable so work resumes (Option A: Resume Continues Work)", () => { + // Pause still emits errorCode agent_paused for observability, but it is NOT + // non-retryable. On resume the agent becomes invokable again and this classifies + // as default/retryable, so the continuation re-enqueues and the issue continues + // rather than escalating to blocked. Durability is guaranteed separately by the + // execution-start guard (Change B), not by this classification. + const c = classifyContinuationFailure(run("agent_paused")); + expect(c.kind).toBe("default"); + expect(c.maxAttempts).toBeGreaterThan(0); + }); + + it("agent_not_invokable (execution-start abort) is non-retryable", () => { + expect(classifyContinuationFailure(run("agent_not_invokable")).kind).toBe("non_retryable"); + }); + + it("timed_out (timeout) still retries as transient infra", () => { + const c = classifyContinuationFailure(run("timeout")); + expect(c.kind).toBe("transient_infra"); + expect(c.maxAttempts).toBeGreaterThan(0); + }); + + it("generic cancelled (non-pause cancellation) is NOT non-retryable", () => { + // non-pause cancellations (the internal invokability cancel and budget pause) keep errorCode "cancelled" -> default branch + expect(classifyContinuationFailure(run("cancelled")).kind).toBe("default"); + }); + + it("genuine failure with no/unknown code retries via default branch", () => { + expect(classifyContinuationFailure(run(null)).kind).toBe("default"); + expect(classifyContinuationFailure(run("some_adapter_error")).kind).toBe("default"); + }); +}); diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 49c01851..f0e33540 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -204,7 +204,7 @@ type ContinuationRetryClassification = { errorCode: string | null; }; -function classifyContinuationFailure(latestRun: LatestIssueRun): ContinuationRetryClassification { +export function classifyContinuationFailure(latestRun: LatestIssueRun): ContinuationRetryClassification { const errorCode = readNonEmptyString(latestRun?.errorCode); if (errorCode && NON_RETRYABLE_CONTINUATION_ERROR_CODES.has(errorCode)) { return { kind: "non_retryable", maxAttempts: 0, baseBackoffMs: 0, errorCode };