[codex] Add clear-error agent action (#7695)

## Thinking Path > - Paperclip is the open source control plane people use to manage AI agents for work. > - Agent runtime state is surfaced in both the server API and the board UI so operators can tell whether an agent is idle, running, paused, or in error. > - When an agent is already in `error`, the existing pause/resume action slot is not useful because there is no running work to pause. > - Operators need a direct, audited recovery path that clears the stale error state only for agents in the same company. > - This pull request adds a company-scoped clear-error mutation, exposes the shared API contract, and wires the board action cluster to show Clear error in the pause/resume slot for errored agents. > - The benefit is that operators can recover CEO/CTO-style errored agents without resorting to database edits or unrelated session reset actions. ## Linked Issues or Issue Description Refs #4021 Paperclip issue: PAP-10515 — right now the CEO and CTO agents are in error state, but there is no way to clear the error; they appear otherwise fine. ## What Changed - Added shared constants, API path, and agent status type support for a company-scoped clear-error action. - Added the server service and route to clear an agent from `error` back to `idle`, with company access enforcement and activity logging. - Added OpenAPI/docs coverage for the clear-error endpoint. - Added backend coverage for service behavior and cross-tenant authorization. - Updated the board agent action cluster to show a red-tinted Clear error button only when `agent.status === "error"`. - Updated agent properties to show a red active last-error indicator only while the agent is currently errored. - Added UI component tests for the error-state action and the non-error pause/resume behavior. ## Verification Local: - `pnpm exec vitest run server/src/__tests__/agents-service-clear-error.test.ts server/src/__tests__/agent-cross-tenant-authz-routes.test.ts ui/src/components/AgentActionButtons.test.tsx` - `pnpm --filter @paperclipai/ui typecheck` - `pnpm exec vitest run server/src/__tests__/openapi-routes.test.ts` PR checks: - Main Paperclip workflow is green on `a7378e584d50594e7bd507a1a02985bfaaa5abf8`. - Greptile is 5/5 with no files requiring special attention and no new comments on the latest review. - `commitperclip PR Review` is still red because its security-gate step canceled after filing a draft advisory; the linked `security-review` check is neutral and says the draft advisory is not a merge block. Visual artifact: - ![Clear error visual comparison](https://gist.githubusercontent.com/cryppadotta/59378c59869b971e92d3edf7b7073aa8/raw/clear-error-visual-comparison.svg) ## Risks Low to medium risk. The mutation is intentionally narrow, but reviewers should check that clearing `lastError`/`lastRunError` and returning to `idle` is the desired recovery semantics for every adapter state. The remaining red check is from the external commitperclip security-review workflow, not from the code/test workflow for this PR. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used OpenAI Codex, GPT-5-family coding model, tool-assisted with local shell, git, GitHub CLI, and targeted Vitest execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge
2026-06-07 10:33:29 -05:00
parent 35c31ca63f
commit 2d1b531a49
15 changed files with 683 additions and 10 deletions
@@ -50,6 +50,7 @@ const mockAgentService = vi.hoisted(() => ({
  getById: vi.fn(),
  pause: vi.fn(),
  resume: vi.fn(),
+  clearError: vi.fn(),
  terminate: vi.fn(),
  remove: vi.fn(),
  listKeys: vi.fn(),
@@ -276,6 +277,7 @@ function resetMockDefaults() {
  mockAgentService.getById.mockImplementation(async () => ({ ...baseAgent }));
  mockAgentService.pause.mockImplementation(async () => ({ ...baseAgent }));
  mockAgentService.resume.mockImplementation(async () => ({ ...baseAgent }));
+  mockAgentService.clearError.mockImplementation(async () => ({ ...baseAgent, status: "idle" }));
  mockAgentService.terminate.mockImplementation(async () => ({ ...baseAgent }));
  mockAgentService.remove.mockImplementation(async () => ({ ...baseAgent }));
  mockAgentService.listKeys.mockImplementation(async () => []);
@@ -334,6 +336,12 @@ describe.sequential("agent cross-tenant route authorization", () => {
          requestApp(app, (baseUrl) => request(baseUrl).post(`/api/agents/${agentId}/pause`).send({})),
        untouched: [mockAgentService.pause, mockHeartbeatService.cancelActiveForAgent],
      },
+      {
+        label: "clear error",
+        request: (app: express.Express) =>
+          requestApp(app, (baseUrl) => request(baseUrl).post(`/api/agents/${agentId}/clear-error`).send({})),
+        untouched: [mockAgentService.clearError],
+      },
      {
        label: "list keys",
        request: (app: express.Express) =>
@@ -386,4 +394,117 @@ describe.sequential("agent cross-tenant route authorization", () => {
    expect(mockAgentService.getKeyById).toHaveBeenCalledWith(keyId);
    expect(mockAgentService.revokeKey).not.toHaveBeenCalled();
  });
+
+  it("requires board access before clearing an agent error", async () => {
+    const app = await createApp({
+      type: "agent",
+      agentId,
+      companyId,
+      runId: "run-1",
+    });
+
+    const res = await requestApp(app, (baseUrl) =>
+      request(baseUrl).post(`/api/agents/${agentId}/clear-error`).send({}),
+    );
+
+    expect(res.status).toBe(403);
+    expect(res.body.error).toContain("Board access required");
+    expect(mockAgentService.clearError).not.toHaveBeenCalled();
+  });
+
+  it("clears error agents and records a distinct audit action", async () => {
+    const errorAgent = {
+      ...baseAgent,
+      status: "error",
+      pauseReason: "system",
+      pausedAt: new Date("2026-04-11T00:02:00.000Z"),
+    };
+    mockAgentService.getById.mockImplementation(async () => ({ ...errorAgent }));
+    mockAgentService.clearError.mockImplementation(async () => ({
+      ...errorAgent,
+      status: "idle",
+      pauseReason: null,
+      pausedAt: null,
+      updatedAt: new Date("2026-04-11T00:03:00.000Z"),
+    }));
+    const app = await createApp({
+      type: "board",
+      userId: "board-user",
+      companyIds: [companyId],
+      source: "local_implicit",
+      isInstanceAdmin: true,
+    });
+
+    const res = await requestApp(app, (baseUrl) =>
+      request(baseUrl).post(`/api/agents/${agentId}/clear-error`).send({}),
+    );
+
+    expect(res.status).toBe(200);
+    expect(res.body).toMatchObject({
+      id: agentId,
+      status: "idle",
+      pauseReason: null,
+      pausedAt: null,
+    });
+    expect(mockAgentService.clearError).toHaveBeenCalledWith(agentId);
+    expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
+      companyId,
+      actorType: "user",
+      actorId: "board-user",
+      action: "agent.error_cleared",
+      entityType: "agent",
+      entityId: agentId,
+    }));
+  });
+
+  it("returns 409 and does not mutate when the agent org chain is invalid", async () => {
+    mockAgentService.getById.mockImplementation(async () => ({
+      ...baseAgent,
+      status: "error",
+      orgChainHealth: {
+        status: "invalid_org_chain",
+        reason: "missing_manager",
+        repairGuidance: "Repair the reporting chain first.",
+      },
+    }));
+    const app = await createApp({
+      type: "board",
+      userId: "board-user",
+      companyIds: [companyId],
+      source: "local_implicit",
+      isInstanceAdmin: true,
+    });
+
+    const res = await requestApp(app, (baseUrl) =>
+      request(baseUrl).post(`/api/agents/${agentId}/clear-error`).send({}),
+    );
+
+    expect(res.status).toBe(409);
+    expect(res.body.error).toContain("Repair the reporting chain first");
+    expect(mockAgentService.clearError).not.toHaveBeenCalled();
+    expect(mockLogActivity).not.toHaveBeenCalled();
+  });
+
+  it("returns a clear 409 for non-error agents", async () => {
+    const { conflict } = await import("../errors.js");
+    mockAgentService.getById.mockImplementation(async () => ({ ...baseAgent, status: "idle" }));
+    mockAgentService.clearError.mockImplementation(async () => {
+      throw conflict("Only agents in error status can have their error cleared");
+    });
+    const app = await createApp({
+      type: "board",
+      userId: "board-user",
+      companyIds: [companyId],
+      source: "local_implicit",
+      isInstanceAdmin: true,
+    });
+
+    const res = await requestApp(app, (baseUrl) =>
+      request(baseUrl).post(`/api/agents/${agentId}/clear-error`).send({}),
+    );
+
+    expect(res.status).toBe(409);
+    expect(res.body.error).toBe("Only agents in error status can have their error cleared");
+    expect(mockLogActivity).not.toHaveBeenCalled();
+  });
 });
@@ -0,0 +1,236 @@
+import { randomUUID } from "node:crypto";
+import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
+import { eq } from "drizzle-orm";
+import {
+  agents,
+  agentRuntimeState,
+  companies,
+  createDb,
+  heartbeatRunEvents,
+  heartbeatRuns,
+} from "@paperclipai/db";
+import {
+  getEmbeddedPostgresTestSupport,
+  startEmbeddedPostgresTestDatabase,
+} from "./helpers/embedded-postgres.js";
+import { agentService } from "../services/agents.ts";
+
+const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
+const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
+
+if (!embeddedPostgresSupport.supported) {
+  console.warn(
+    `Skipping embedded Postgres agent service tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
+  );
+}
+
+describeEmbeddedPostgres("agent service clearError", () => {
+  let db!: ReturnType<typeof createDb>;
+  let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
+
+  beforeAll(async () => {
+    tempDb = await startEmbeddedPostgresTestDatabase("paperclip-agent-clear-error-");
+    db = createDb(tempDb.connectionString);
+  }, 20_000);
+
+  afterEach(async () => {
+    await db.delete(heartbeatRunEvents);
+    await db.delete(agentRuntimeState);
+    await db.delete(heartbeatRuns);
+    await db.delete(agents);
+    await db.delete(companies);
+  });
+
+  afterAll(async () => {
+    await tempDb?.cleanup();
+  });
+
+  it("moves an error agent to idle without deleting run history or runtime diagnostics", async () => {
+    const companyId = randomUUID();
+    const agentId = randomUUID();
+    const runId = randomUUID();
+    const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
+
+    await db.insert(companies).values({
+      id: companyId,
+      name: "Paperclip",
+      issuePrefix,
+      requireBoardApprovalForNewAgents: false,
+    });
+
+    await db.insert(agents).values({
+      id: agentId,
+      companyId,
+      name: "CodexCoder",
+      role: "engineer",
+      status: "error",
+      pauseReason: "system",
+      pausedAt: new Date("2026-06-07T00:00:00.000Z"),
+      adapterType: "codex_local",
+      adapterConfig: {},
+      runtimeConfig: {},
+      permissions: {},
+    });
+
+    await db.insert(heartbeatRuns).values({
+      id: runId,
+      companyId,
+      agentId,
+      invocationSource: "on_demand",
+      status: "failed",
+      error: "Adapter exited with code 1",
+      stdoutExcerpt: "stdout stays inspectable",
+      stderrExcerpt: "stderr stays inspectable",
+      logStore: "local_disk",
+      logRef: "runs/failed.log",
+      resultJson: { sessionId: "codex-session-1" },
+      finishedAt: new Date("2026-06-07T00:01:00.000Z"),
+    });
+
+    await db.insert(heartbeatRunEvents).values({
+      companyId,
+      runId,
+      agentId,
+      seq: 1,
+      eventType: "transcript",
+      stream: "stderr",
+      message: "transcript stays inspectable",
+      payload: { itemType: "error" },
+    });
+
+    await db.insert(agentRuntimeState).values({
+      agentId,
+      companyId,
+      adapterType: "codex_local",
+      sessionId: "codex-session-1",
+      stateJson: { taskKey: "issue:test" },
+      lastRunId: runId,
+      lastRunStatus: "failed",
+      lastError: "Adapter exited with code 1",
+    });
+
+    const cleared = await agentService(db).clearError(agentId);
+
+    expect(cleared).toMatchObject({
+      id: agentId,
+      status: "idle",
+      pauseReason: null,
+      pausedAt: null,
+    });
+
+    const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
+    expect(run).toMatchObject({
+      id: runId,
+      status: "failed",
+      error: "Adapter exited with code 1",
+      stdoutExcerpt: "stdout stays inspectable",
+      stderrExcerpt: "stderr stays inspectable",
+      logStore: "local_disk",
+      logRef: "runs/failed.log",
+    });
+
+    const transcriptEvents = await db
+      .select()
+      .from(heartbeatRunEvents)
+      .where(eq(heartbeatRunEvents.runId, runId));
+    expect(transcriptEvents).toHaveLength(1);
+    expect(transcriptEvents[0]).toMatchObject({
+      runId,
+      eventType: "transcript",
+      stream: "stderr",
+      message: "transcript stays inspectable",
+      payload: { itemType: "error" },
+    });
+
+    const [runtimeState] = await db
+      .select()
+      .from(agentRuntimeState)
+      .where(eq(agentRuntimeState.agentId, agentId));
+    expect(runtimeState).toMatchObject({
+      agentId,
+      sessionId: "codex-session-1",
+      stateJson: { taskKey: "issue:test" },
+      lastRunId: runId,
+      lastRunStatus: "failed",
+      lastError: "Adapter exited with code 1",
+    });
+  });
+
+  it("rejects non-error agents with a 409 conflict", async () => {
+    const companyId = randomUUID();
+    const agentId = randomUUID();
+    const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
+
+    await db.insert(companies).values({
+      id: companyId,
+      name: "Paperclip",
+      issuePrefix,
+      requireBoardApprovalForNewAgents: false,
+    });
+
+    await db.insert(agents).values({
+      id: agentId,
+      companyId,
+      name: "CodexCoder",
+      role: "engineer",
+      status: "idle",
+      adapterType: "codex_local",
+      adapterConfig: {},
+      runtimeConfig: {},
+      permissions: {},
+    });
+
+    await expect(agentService(db).clearError(agentId)).rejects.toMatchObject({
+      status: 409,
+      message: "Only agents in error status can have their error cleared",
+    });
+  });
+
+  it("keeps resume-style terminal and pending-approval protections", async () => {
+    const companyId = randomUUID();
+    const terminatedAgentId = randomUUID();
+    const pendingAgentId = randomUUID();
+    const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
+
+    await db.insert(companies).values({
+      id: companyId,
+      name: "Paperclip",
+      issuePrefix,
+      requireBoardApprovalForNewAgents: false,
+    });
+
+    await db.insert(agents).values([
+      {
+        id: terminatedAgentId,
+        companyId,
+        name: "Terminated",
+        role: "engineer",
+        status: "terminated",
+        adapterType: "codex_local",
+        adapterConfig: {},
+        runtimeConfig: {},
+        permissions: {},
+      },
+      {
+        id: pendingAgentId,
+        companyId,
+        name: "Pending",
+        role: "engineer",
+        status: "pending_approval",
+        adapterType: "codex_local",
+        adapterConfig: {},
+        runtimeConfig: {},
+        permissions: {},
+      },
+    ]);
+
+    await expect(agentService(db).clearError(terminatedAgentId)).rejects.toMatchObject({
+      status: 409,
+      message: "Cannot clear error on terminated agent",
+    });
+    await expect(agentService(db).clearError(pendingAgentId)).rejects.toMatchObject({
+      status: 409,
+      message: "Pending approval agents cannot have errors cleared",
+    });
+  });
+});
@@ -2879,6 +2879,38 @@ export function agentRoutes(
    res.json(agent);
  });

+  router.post("/agents/:id/clear-error", async (req, res) => {
+    assertBoard(req);
+    const id = req.params.id as string;
+    const existing = await getAccessibleAgent(req, res, id);
+    if (!existing) {
+      return;
+    }
+    if (existing.orgChainHealth?.status === "invalid_org_chain") {
+      res.status(409).json({
+        error: existing.orgChainHealth?.repairGuidance ?? "Repair this agent's reporting chain before clearing its error",
+      });
+      return;
+    }
+
+    const agent = await svc.clearError(id);
+    if (!agent) {
+      res.status(404).json({ error: "Agent not found" });
+      return;
+    }
+
+    await logActivity(db, {
+      companyId: agent.companyId,
+      actorType: "user",
+      actorId: req.actor.userId ?? "board",
+      action: "agent.error_cleared",
+      entityType: "agent",
+      entityId: agent.id,
+    });
+
+    res.json(agent);
+  });
+
  router.post("/agents/:id/approve", async (req, res) => {
    assertBoard(req);
    const id = req.params.id as string;
@@ -418,6 +418,10 @@ const responses = {
    description: "Not found",
    content: { "application/json": { schema: ErrorSchema } },
  },
+  conflict: {
+    description: "Conflict",
+    content: { "application/json": { schema: ErrorSchema } },
+  },
  unprocessable: {
    description: "Unprocessable entity",
    content: { "application/json": { schema: ErrorSchema } },
@@ -1247,6 +1251,15 @@ registry.registerPath({
  responses: { 200: r.ok(), 401: r.unauthorized },
 });

+registry.registerPath({
+  method: "post",
+  path: "/api/agents/{id}/clear-error",
+  tags: ["agents"],
+  summary: "Clear an agent error",
+  request: { params: z.object({ id: z.string() }) },
+  responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound, 409: r.conflict },
+});
+
 registry.registerPath({
  method: "post",
  path: "/api/agents/{id}/terminate",
@@ -526,6 +526,35 @@ export function agentService(db: Db) {
      return updated ? getById(updated.id) : null;
    },

+    clearError: async (id: string) => {
+      const existing = await getById(id);
+      if (!existing) return null;
+      if (existing.status === "terminated") throw conflict("Cannot clear error on terminated agent");
+      if (existing.status === "pending_approval") {
+        throw conflict("Pending approval agents cannot have errors cleared");
+      }
+      if (existing.status !== "error") {
+        throw conflict("Only agents in error status can have their error cleared");
+      }
+
+      const updated = await db
+        .update(agents)
+        .set({
+          status: "idle",
+          pauseReason: null,
+          pausedAt: null,
+          updatedAt: new Date(),
+        })
+        .where(and(eq(agents.id, id), eq(agents.status, "error")))
+        .returning()
+        .then((rows) => rows[0] ?? null);
+
+      if (!updated) {
+        throw conflict("Only agents in error status can have their error cleared");
+      }
+      return getById(updated.id);
+    },
+
    terminate: async (id: string) => {
      const existing = await getById(id);
      if (!existing) return null;