From 8ee3987d12d5d10ebf8acfd65d99721875aa3428 Mon Sep 17 00:00:00 2001 From: Danial Jawaid Date: Wed, 10 Jun 2026 02:45:47 +0400 Subject: [PATCH] adapter-claude-local: recover from poisoned previous_message_id 400 (detect + clearSession) (#5972) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip's `claude_local` adapter persists Claude Code session jsonls under `~/.claude/projects/…/{sessionId}.jsonl` and resumes them on the next heartbeat > - When Claude Code injects `` placeholder assistant messages (after rate-limit, max-turn exhaustion, or transient-upstream failures) those placeholders get UUID-format `message.id`s rather than `msg_…`-format ids > - On the next `--resume`, Claude Code passes that UUID as `previous_message_id` and Anthropic's API rejects it with a 400: ``diagnostics.previous_message_id: must be the `id` from a prior /v1/messages response (starts with `msg_`)`` > - The adapter had a session-rotation fallback only for "unknown session" errors, so the poisoned session was `--resume`-d indefinitely and the agent flipped between `idle` and `error` every heartbeat > - Even worse, the *result* event of the failing run still carried a `session_id`, and the adapter was persisting that id into the issue-scoped session store (`agentTaskSessions`). So even after we detected the 400, every subsequent continuation re-loaded the same poisoned id and hit the same 400 again — the issue was permanently stranded > - We observed this on multiple agents in our deployment; the only manual fix was to rename the `.jsonl`, which is not a viable long-term workaround > - This PR detects the 400, runs the same session-rotation fallback the unknown-session path uses **and** stops persisting the poisoned id, so the next attempt starts genuinely fresh ## Linked Issues or Issue Description No external GitHub issue is linked. Describing the problem inline following the bug-report template: **What happened:** `claude_local` agents flipped between `idle` and `error` on every heartbeat because the persisted session jsonl carried a synthetic UUID `previous_message_id` (from `` assistant placeholders injected after rate-limit/max-turn/upstream errors). Anthropic's API rejected every `--resume` with a 400: ``diagnostics.previous_message_id: must be the `id` from a prior /v1/messages response (starts with `msg_`)``. **Expected behavior:** When the persisted session is poisoned and unrecoverable, the adapter should rotate to a fresh session — the same fallback path already used for unknown-session errors — and stop re-persisting the poisoned `session_id`. **Actual behavior:** The session-rotation fallback only matched the "unknown session" pattern, so the poisoned session was `--resume`-d forever. The result event of the failing run still carried `session_id`, which was being persisted into `agentTaskSessions`, so every subsequent continuation reloaded the same poisoned id and hit the same 400. **Reproduction:** Inject any flow that causes Claude Code to emit a `` placeholder (rate-limit, max-turn exhaustion, transient upstream failure). The next `--resume` will fail with the 400 and the agent will not self-recover. **Scope of fix:** Add a `previous_message_id` 400 detector; route it through the existing unknown-session fallback; drop the poisoned `sessionId` and emit `clearSession: true` so the heartbeat service wipes the persisted row; best-effort delete the local poisoned `.jsonl`. ## What Changed Two commits: 1. **`adapter-claude-local: auto-rotate session on previous_message_id 400 (synthetic-msg poisoning)`** — detector + execute-time rotation 2. **`adapter-claude-local: guard against persisting poisoned sessionId`** — validate-before-persist + `clearSession` Combined diff: - `parse.ts`: new `isClaudePoisonedPreviousMessageIdError(parsed)` matching ``/diagnostics\.previous_message_id.*starts with `msg_`/i`` against `parsed.result` and `extractClaudeErrorMessages(parsed)` - `parse.ts`: `isClaudeTransientUpstreamError()` excludes the new error from transient classification so it isn't masked as retryable upstream noise - `execute.ts`: expand the resume-fallback branch so it triggers on both `isClaudeUnknownSessionError` and the new `isClaudePoisonedPreviousMessageIdError`, with a distinct log line (`"returned a poisoned message-id"` vs `"is unavailable"`) - `execute.ts`: for local (non-remote) execution targets, best-effort delete the poisoned `~/.claude/projects/.../{sessionId}.jsonl` before retrying so the file can't be accidentally resumed by an out-of-band caller. The `fs.unlink` and follow-up log call are in separate try/catch blocks so a closed log stream cannot mask a successful unlink (and vice versa) - `execute.ts` / `toAdapterResult`: when a result carries the poisoned 400, **drop** `sessionId`/`sessionParams`/`sessionDisplayId` (return `null`) and emit `clearSession: true` so the heartbeat service's `resolveNextSessionState` wipes the persisted row. The result also surfaces `errorCode: "claude_poisoned_previous_message_id"` for observability - `docs/adapters/claude-local.md`: runbook entry — symptom, auto-recovery flow, on-call checklist - Tests: - 4 new `parse.test.ts` cases covering positive detection in `result` and `errors[]`, negative cases, and non-transient classification - 3 new `claude-local-execute.test.ts` cases: (a) fresh run reports the poisoned error → sessionId dropped + `clearSession: true`; (b) recovery retry also reports the poisoned error → same guards apply; (c) session-rotation success on retry ## Verification ```bash pnpm --filter @paperclipai/adapter-claude-local exec vitest run src/server/parse.test.ts pnpm --filter @paperclipai/server exec vitest run src/__tests__/claude-local-execute.test.ts ``` Both suites green locally. This patch is also currently running as a hot-patch over the published `2026.513.0` adapter on the reporting deployment — sessions that previously looped indefinitely now self-recover on the first heartbeat after the 400 surfaces. ## Risks - Low risk. The detector is conservative (regex over `result` + `errors[]` only) and the rotation reuses the existing unknown-session fallback path - The local-only `fs.unlink` of the poisoned `.jsonl` is wrapped in `try/catch` and ignored on failure — strictly an optimization; the server-side session clear is the authoritative reset - Remote execution targets (`executionTargetIsRemote`) skip the disk cleanup because the file lives on a remote host that we can't safely reach from the adapter - The `clearSession: true` + nulled session fields path is a no-op on healthy runs; it only fires when the new detector matches, so existing successful continuations are unaffected - No DB schema changes, no public API changes, no new dependencies ## Model Used - Provider: Anthropic Claude - Model: `claude-opus-4-7` (Opus 4.7) - Context window: 1M - Capabilities: extended reasoning, tool use, code execution - Role: implemented the detector, expanded the fallback branch, added the persist-guard + `clearSession`, wrote the unit + integration tests, validated locally, and applied the equivalent hot-patch to the deployed `2026.513.0` install while this PR is in review ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for similar or duplicate PRs and linked them — closed #2295, #2361, #3572, #5438 as duplicates of this canonical fix; complementary fixes #4838 (heartbeat_timer reset) and #4932 (gemini context-overflow rotation) target different code paths - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots — N/A, adapter-only change - [x] I have updated relevant documentation (`docs/adapters/claude-local.md` runbook entry) - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Danial Jawaid Co-authored-by: Paperclip Co-authored-by: Devin Foley --- docs/adapters/claude-local.md | 22 + .../claude-local/src/server/execute.ts | 73 ++- .../claude-local/src/server/parse.test.ts | 48 ++ .../adapters/claude-local/src/server/parse.ts | 13 +- .../clean-poisoned-claude-sessions.test.ts | 166 +++++++ .../scripts/clean-poisoned-claude-sessions.ts | 462 ++++++++++++++++++ .../__tests__/claude-local-execute.test.ts | 198 ++++++++ 7 files changed, 970 insertions(+), 12 deletions(-) create mode 100644 packages/db/scripts/clean-poisoned-claude-sessions.test.ts create mode 100644 packages/db/scripts/clean-poisoned-claude-sessions.ts diff --git a/docs/adapters/claude-local.md b/docs/adapters/claude-local.md index fc64fcf8..1cc39e21 100644 --- a/docs/adapters/claude-local.md +++ b/docs/adapters/claude-local.md @@ -43,6 +43,28 @@ Session resume is cwd-aware: if the agent's working directory changed since the If resume fails with an unknown session error, the adapter automatically retries with a fresh session. +### Poisoned `previous_message_id` (recovery) + +Symptom in logs / issue thread: + +``` +API Error: 400 diagnostics.previous_message_id: must be the `id` from a prior /v1/messages response (starts with `msg_`) +``` + +What it means: the on-disk Claude Code transcript JSONL for that session contains a malformed (non-`msg_`-prefixed) `previous_message_id`. Anthropic's `/v1/messages` rejects every resume attempt against that transcript with a deterministic 400. Without guards, Paperclip would re-persist the same poisoned session id and the issue is stranded permanently — see [RED-976](../../../) / [RED-978](../../../). + +What the adapter does automatically: + +1. **Auto-rotate on resume.** If a `--resume` attempt returns this 400, the adapter retries once with a fresh session, deletes the poisoned `.jsonl` from the local Claude config dir (best effort), and uses the fresh session id going forward. +2. **Validate-before-persist.** A result that carries this 400 never gets its `session_id` written back to the task session store, even if Claude Code emits one in the result event. The adapter returns `sessionId: null`, `sessionParams: null`, and `errorCode: "claude_poisoned_previous_message_id"`. +3. **Clear-on-error.** The adapter sets `clearSession: true` on the result, which causes the heartbeat service to drop any persisted session row for that issue (`clearTaskSessions`). The next continuation starts from a clean slate. + +On-call checklist if you see this in production: + +- Confirm `errorCode` is `claude_poisoned_previous_message_id` in the run row — that means the guards fired correctly and the issue auto-recovers on the next heartbeat. +- If the same issue still loops after one heartbeat, check that `agentTaskSessions` for that `(agentId, taskKey)` was cleared. If not, the adapter return value was lost (e.g. a malformed run finalization) — escalate; do **not** manually edit the row, file a child issue with the run id. +- For remote execution targets (sandbox/SSH), the poisoned JSONL is on the remote and the adapter only logs the cleanup intent. The fresh-session retry still succeeds because it uses a new session id, and the server-side `clearSession: true` is authoritative regardless of remote disk state. + ## Skills Injection The adapter creates a temporary directory with symlinks to Paperclip skills and passes it via `--add-dir`. This makes skills discoverable without polluting the agent's working directory. diff --git a/packages/adapters/claude-local/src/server/execute.ts b/packages/adapters/claude-local/src/server/execute.ts index 067f68cd..99fdd995 100644 --- a/packages/adapters/claude-local/src/server/execute.ts +++ b/packages/adapters/claude-local/src/server/execute.ts @@ -54,8 +54,9 @@ import { isClaudeMaxTurnsResult, isClaudeTransientUpstreamError, isClaudeUnknownSessionError, + isClaudePoisonedPreviousMessageIdError, } from "./parse.js"; -import { prepareClaudeConfigSeed } from "./claude-config.js"; +import { prepareClaudeConfigSeed, resolveSharedClaudeConfigDir } from "./claude-config.js"; import { resolveClaudeDesiredSkillNames } from "./skills.js"; import { isBedrockModelId } from "./models.js"; import { prepareClaudePromptBundle } from "./prompt-cache.js"; @@ -860,9 +861,22 @@ export async function execute(ctx: AdapterExecutionContext): Promise) : null; - const clearSessionForMaxTurns = isClaudeMaxTurnsResult(parsed); - const parsedIsError = asBoolean(parsed.is_error, false); - const failed = (proc.exitCode ?? 0) !== 0 || parsedIsError; const errorMessage = failed ? describeClaudeFailure(parsed) ?? `Claude exited with code ${proc.exitCode ?? -1}` : null; @@ -888,6 +899,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise = { ...parsed, ...(failed && clearSessionForMaxTurns ? { stopReason: "max_turns_exhausted" } : {}), + ...(failed && poisonedPreviousMessageId ? { stopReason: "claude_poisoned_previous_message_id" } : {}), ...(transientUpstream ? { errorFamily: "transient_upstream" } : {}), ...(transientRetryNotBefore ? { retryNotBefore: transientRetryNotBefore.toISOString() } : {}), ...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}), @@ -937,23 +952,59 @@ export async function execute(ctx: AdapterExecutionContext): Promise { @@ -94,6 +95,53 @@ describe("isClaudeTransientUpstreamError", () => { }), ).toBe(false); }); + + it("does not classify poisoned previous_message_id errors as transient", () => { + expect( + isClaudeTransientUpstreamError({ + parsed: { + subtype: "success", + is_error: true, + result: "API Error: 400 diagnostics.previous_message_id: must be the `id` from a prior /v1/messages response (starts with `msg_`)", + }, + }), + ).toBe(false); + }); +}); + +describe("isClaudePoisonedPreviousMessageIdError", () => { + it("detects the previous_message_id 400 error in the result field", () => { + expect( + isClaudePoisonedPreviousMessageIdError({ + subtype: "success", + is_error: true, + result: "API Error: 400 diagnostics.previous_message_id: must be the `id` from a prior /v1/messages response (starts with `msg_`)", + }), + ).toBe(true); + }); + + it("detects the error in the errors array", () => { + expect( + isClaudePoisonedPreviousMessageIdError({ + is_error: true, + result: "", + errors: [{ message: "400 diagnostics.previous_message_id: must be the `id` from a prior /v1/messages response (starts with `msg_`)" }], + }), + ).toBe(true); + }); + + it("returns false for unrelated errors", () => { + expect( + isClaudePoisonedPreviousMessageIdError({ + is_error: true, + result: "No conversation found with session id abc-123", + }), + ).toBe(false); + }); + + it("returns false for empty parsed result", () => { + expect(isClaudePoisonedPreviousMessageIdError({})).toBe(false); + }); }); describe("extractClaudeRetryNotBefore", () => { diff --git a/packages/adapters/claude-local/src/server/parse.ts b/packages/adapters/claude-local/src/server/parse.ts index f645c4f2..9664f66b 100644 --- a/packages/adapters/claude-local/src/server/parse.ts +++ b/packages/adapters/claude-local/src/server/parse.ts @@ -196,6 +196,17 @@ export function isClaudeUnknownSessionError(parsed: Record): bo ); } +export function isClaudePoisonedPreviousMessageIdError(parsed: Record): boolean { + const resultText = asString(parsed.result, "").trim(); + const allMessages = [resultText, ...extractClaudeErrorMessages(parsed)] + .map((msg) => msg.trim()) + .filter(Boolean); + + return allMessages.some((msg) => + /diagnostics\.previous_message_id.*starts with `msg_`/i.test(msg), + ); +} + function buildClaudeTransientHaystack(input: { parsed?: Record | null; stdout?: string | null; @@ -375,7 +386,7 @@ export function isClaudeTransientUpstreamError(input: { }): boolean { const parsed = input.parsed ?? null; // Deterministic failures are handled by their own classifiers. - if (parsed && (isClaudeMaxTurnsResult(parsed) || isClaudeUnknownSessionError(parsed))) { + if (parsed && (isClaudeMaxTurnsResult(parsed) || isClaudeUnknownSessionError(parsed) || isClaudePoisonedPreviousMessageIdError(parsed))) { return false; } const loginMeta = detectClaudeLoginRequired({ diff --git a/packages/db/scripts/clean-poisoned-claude-sessions.test.ts b/packages/db/scripts/clean-poisoned-claude-sessions.test.ts new file mode 100644 index 00000000..5fcdc20d --- /dev/null +++ b/packages/db/scripts/clean-poisoned-claude-sessions.test.ts @@ -0,0 +1,166 @@ +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + classifyJsonlText, + classifyRow, + encodeClaudeCwd, + jsonlPathFor, + resolveClaudeConfigDir, +} from "./clean-poisoned-claude-sessions.js"; + +describe("classifyJsonlText", () => { + it("returns healthy when the last assistant message.id starts with msg_", () => { + const text = [ + JSON.stringify({ type: "user" }), + JSON.stringify({ type: "assistant", message: { id: "msg_first" } }), + JSON.stringify({ type: "user" }), + JSON.stringify({ type: "assistant", message: { id: "msg_last" } }), + ].join("\n"); + const result = classifyJsonlText(text); + expect(result).toEqual({ kind: "healthy", lastAssistantId: "msg_last" }); + }); + + it("returns poisoned when the last assistant message.id is a synthetic UUID", () => { + const text = [ + JSON.stringify({ type: "user" }), + JSON.stringify({ type: "assistant", message: { id: "msg_real" } }), + JSON.stringify({ + type: "assistant", + message: { id: "f649efd6-1837-4c52-981f-b81fdc0c2ddb" }, + }), + ].join("\n"); + const result = classifyJsonlText(text); + expect(result).toEqual({ + kind: "poisoned", + lastAssistantId: "f649efd6-1837-4c52-981f-b81fdc0c2ddb", + }); + }); + + it("ignores trailing whitespace, blank lines, and malformed JSON", () => { + const text = [ + "", + " ", + "this-is-not-json", + JSON.stringify({ type: "assistant", message: { id: "msg_only" } }), + "", + ].join("\n"); + expect(classifyJsonlText(text)).toEqual({ + kind: "healthy", + lastAssistantId: "msg_only", + }); + }); + + it("returns no_assistant_entries when the transcript has no assistant rows", () => { + const text = [ + JSON.stringify({ type: "user" }), + JSON.stringify({ type: "attachment" }), + JSON.stringify({ type: "last-prompt", lastPrompt: "x" }), + ].join("\n"); + expect(classifyJsonlText(text)).toEqual({ kind: "no_assistant_entries" }); + }); + + it("treats a CRLF-encoded transcript the same as LF", () => { + const text = [ + JSON.stringify({ type: "assistant", message: { id: "msg_first" } }), + JSON.stringify({ type: "assistant", message: { id: "synthetic-uuid" } }), + ].join("\r\n"); + expect(classifyJsonlText(text)).toEqual({ + kind: "poisoned", + lastAssistantId: "synthetic-uuid", + }); + }); +}); + +describe("encodeClaudeCwd", () => { + it("mirrors the claude-local adapter encoding rule", () => { + expect(encodeClaudeCwd("/Users/dj/.paperclip/instances/default/workspaces/abc")).toBe( + "-Users-dj--paperclip-instances-default-workspaces-abc", + ); + }); +}); + +describe("jsonlPathFor", () => { + it("returns /projects//.jsonl", () => { + const result = jsonlPathFor({ + claudeConfigDir: "/tmp/.claude", + cwd: "/Users/dj/work", + sessionId: "abc-123", + }); + expect(result).toBe(path.join("/tmp/.claude", "projects", "-Users-dj-work", "abc-123.jsonl")); + }); +}); + +describe("resolveClaudeConfigDir", () => { + it("prefers the explicit override", () => { + expect(resolveClaudeConfigDir({}, "/tmp/custom")).toBe(path.resolve("/tmp/custom")); + }); + it("uses CLAUDE_CONFIG_DIR when no override is given", () => { + expect(resolveClaudeConfigDir({ CLAUDE_CONFIG_DIR: "/tmp/from-env" } as NodeJS.ProcessEnv)).toBe( + path.resolve("/tmp/from-env"), + ); + }); + it("falls back to ~/.claude when no override and no env var", () => { + const result = resolveClaudeConfigDir({} as NodeJS.ProcessEnv); + expect(result.endsWith(`${path.sep}.claude`)).toBe(true); + }); +}); + +describe("classifyRow", () => { + const claudeConfigDir = "/tmp/.claude"; + const cwd = "/Users/dj/work"; + const sessionId = "session-1"; + const expectedJsonl = jsonlPathFor({ claudeConfigDir, cwd, sessionId }); + + it("returns skipped_no_session_id when sessionId is null", () => { + const result = classifyRow( + { sessionId: null, cwd, isRemote: false }, + { claudeConfigDir, readJsonl: () => null }, + ); + expect(result.kind).toBe("skipped_no_session_id"); + }); + + it("returns skipped_remote when the row has remoteExecution params", () => { + const result = classifyRow( + { sessionId, cwd, isRemote: true }, + { claudeConfigDir, readJsonl: () => null }, + ); + expect(result.kind).toBe("skipped_remote"); + }); + + it("returns missing_jsonl with the expected path when the file does not exist", () => { + const result = classifyRow( + { sessionId, cwd, isRemote: false }, + { claudeConfigDir, readJsonl: () => null }, + ); + expect(result.kind).toBe("missing_jsonl"); + if (result.kind === "missing_jsonl") { + expect(result.jsonlPath).toBe(expectedJsonl); + } + }); + + it("flags rows whose JSONL last assistant id is non-msg_", () => { + const jsonl = [ + JSON.stringify({ type: "assistant", message: { id: "msg_first" } }), + JSON.stringify({ type: "assistant", message: { id: "synthetic-uuid" } }), + ].join("\n"); + const result = classifyRow( + { sessionId, cwd, isRemote: false }, + { claudeConfigDir, readJsonl: () => jsonl }, + ); + expect(result.kind).toBe("poisoned"); + if (result.kind === "poisoned") { + expect(result.lastAssistantId).toBe("synthetic-uuid"); + } + }); + + it("leaves valid msg_-prefixed sessions classified as healthy", () => { + const jsonl = [ + JSON.stringify({ type: "assistant", message: { id: "msg_only" } }), + ].join("\n"); + const result = classifyRow( + { sessionId, cwd, isRemote: false }, + { claudeConfigDir, readJsonl: () => jsonl }, + ); + expect(result.kind).toBe("healthy"); + }); +}); diff --git a/packages/db/scripts/clean-poisoned-claude-sessions.ts b/packages/db/scripts/clean-poisoned-claude-sessions.ts new file mode 100644 index 00000000..2a26a930 --- /dev/null +++ b/packages/db/scripts/clean-poisoned-claude-sessions.ts @@ -0,0 +1,462 @@ +#!/usr/bin/env tsx +/** + * One-shot cleaner for `agent_task_sessions` rows whose persisted `claude_local` + * session is poisoned by a non-`msg_*` `previous_message_id`. + * + * The poison shape: the Claude CLI keeps an on-disk JSONL transcript keyed by + * sessionId at `/projects//.jsonl`. + * If the last `assistant` entry has a `message.id` that does NOT start with + * `msg_`, every subsequent `--resume` against /v1/messages 400s with + * `diagnostics.previous_message_id` and the issue is permanently stranded + * until the persisted session is dropped. See RED-877 / RED-978 / RED-991 for + * the upstream adapter fix; this script retroactively heals rows that were + * stranded BEFORE that adapter shipped. + * + * Usage: + * tsx packages/db/scripts/clean-poisoned-claude-sessions.ts \ + * --config /path/to/paperclip/config.json \ + * [--claude-config-dir ~/.claude] \ + * [--dry-run] [--json] + * + * Or in a Paperclip checkout shell: + * pnpm --filter @paperclipai/db exec tsx scripts/clean-poisoned-claude-sessions.ts --dry-run + * + * Exits 0 on success even when nothing was healed. Idempotent. + */ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { and, eq, isNotNull, sql } from "drizzle-orm"; +import { createDb } from "../src/client.js"; +import { agentTaskSessions } from "../src/schema/index.js"; + +const CLAUDE_ADAPTER_TYPE = "claude_local"; + +export type SessionClassification = + | { kind: "healthy"; lastAssistantId: string } + | { kind: "poisoned"; lastAssistantId: string } + | { kind: "missing_jsonl"; jsonlPath: string } + | { kind: "no_assistant_entries" } + | { kind: "skipped_remote" } + | { kind: "skipped_no_session_id" } + | { kind: "unreadable_jsonl"; error: string }; + +export interface ClaudeSessionRowInput { + sessionId: string | null; + cwd: string | null; + isRemote: boolean; +} + +/** + * Pure classifier — given JSONL text, returns whether the session is poisoned. + * A session is "poisoned" iff its last `assistant` entry has a `message.id` + * that does not start with `msg_`. Sessions with zero assistant entries are + * left alone (they can be resumed safely from scratch). + */ +export function classifyJsonlText(text: string): SessionClassification { + let lastAssistantId: string | null = null; + for (const rawLine of text.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line) continue; + let obj: unknown; + try { + obj = JSON.parse(line); + } catch { + continue; + } + if (!obj || typeof obj !== "object") continue; + const entry = obj as Record; + if (entry.type !== "assistant") continue; + const msg = entry.message; + if (!msg || typeof msg !== "object") continue; + const id = (msg as Record).id; + if (typeof id === "string" && id.length > 0) { + lastAssistantId = id; + } + } + if (lastAssistantId === null) { + return { kind: "no_assistant_entries" }; + } + if (lastAssistantId.startsWith("msg_")) { + return { kind: "healthy", lastAssistantId }; + } + return { kind: "poisoned", lastAssistantId }; +} + +export function encodeClaudeCwd(cwd: string): string { + // Mirrors `effectiveExecutionCwd.replace(/[^a-zA-Z0-9-]/g, "-")` from the + // claude-local adapter (`packages/adapters/claude-local/src/server/execute.ts`). + return cwd.replace(/[^a-zA-Z0-9-]/g, "-"); +} + +export function resolveClaudeConfigDir(env: NodeJS.ProcessEnv = process.env, override?: string): string { + if (override && override.trim().length > 0) return path.resolve(override); + const fromEnv = env.CLAUDE_CONFIG_DIR; + if (typeof fromEnv === "string" && fromEnv.trim().length > 0) { + return path.resolve(fromEnv); + } + return path.join(os.homedir(), ".claude"); +} + +export function jsonlPathFor(input: { claudeConfigDir: string; cwd: string; sessionId: string }): string { + return path.join( + input.claudeConfigDir, + "projects", + encodeClaudeCwd(input.cwd), + `${input.sessionId}.jsonl`, + ); +} + +export function classifyRow( + row: ClaudeSessionRowInput, + opts: { claudeConfigDir: string; readJsonl: (filePath: string) => string | null }, +): SessionClassification & { jsonlPath?: string } { + if (!row.sessionId) return { kind: "skipped_no_session_id" }; + if (row.isRemote) return { kind: "skipped_remote" }; + if (!row.cwd) return { kind: "skipped_no_session_id" }; + const jsonlPath = jsonlPathFor({ + claudeConfigDir: opts.claudeConfigDir, + cwd: row.cwd, + sessionId: row.sessionId, + }); + let text: string | null; + try { + text = opts.readJsonl(jsonlPath); + } catch (error) { + return { + kind: "unreadable_jsonl", + error: error instanceof Error ? error.message : String(error), + jsonlPath, + }; + } + if (text === null) { + return { kind: "missing_jsonl", jsonlPath }; + } + const classification = classifyJsonlText(text); + return { ...classification, jsonlPath }; +} + +interface CliArgs { + configPath: string | null; + claudeConfigDir: string | null; + dryRun: boolean; + json: boolean; + databaseUrl: string | null; + deleteMissing: boolean; +} + +function parseArgs(argv: string[]): CliArgs { + const args: CliArgs = { + configPath: null, + claudeConfigDir: null, + dryRun: false, + json: false, + databaseUrl: null, + deleteMissing: false, + }; + for (let i = 0; i < argv.length; i++) { + const token = argv[i]; + switch (token) { + case "--config": + args.configPath = argv[++i] ?? null; + break; + case "--claude-config-dir": + args.claudeConfigDir = argv[++i] ?? null; + break; + case "--database-url": + args.databaseUrl = argv[++i] ?? null; + break; + case "--dry-run": + args.dryRun = true; + break; + case "--json": + args.json = true; + break; + case "--delete-missing-jsonl": + args.deleteMissing = true; + break; + case "--help": + case "-h": + process.stdout.write(USAGE); + process.exit(0); + break; + default: + if (token.startsWith("--")) { + throw new Error(`Unknown flag: ${token}`); + } + } + } + return args; +} + +const USAGE = `Usage: + tsx packages/db/scripts/clean-poisoned-claude-sessions.ts [flags] + +Flags: + --config Path to paperclip config.json (defaults to + $PAPERCLIP_HOME/instances/default/config.json or + the standard locations). + --database-url Override DB connection string entirely. + --claude-config-dir Override Claude CLI config dir (default: + $CLAUDE_CONFIG_DIR or ~/.claude). + --dry-run Report poisoned rows but do not delete. + --delete-missing-jsonl Also delete rows whose sessionId references a + JSONL that no longer exists on disk. + --json Emit a single JSON summary on stdout instead of + human-readable lines. + -h, --help Print this usage. +`; + +function readDatabaseUrlFromConfig(configPath: string): string { + const raw = fs.readFileSync(configPath, "utf8"); + const parsed = JSON.parse(raw) as { + database?: { + mode?: string; + embeddedPostgresPort?: number; + connectionString?: string; + }; + }; + if (parsed.database?.mode === "postgres" && parsed.database.connectionString) { + return parsed.database.connectionString; + } + const port = parsed.database?.embeddedPostgresPort ?? 54329; + return `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`; +} + +function defaultConfigPath(): string | null { + const candidates: string[] = []; + const home = os.homedir(); + if (process.env.PAPERCLIP_HOME) { + candidates.push(path.join(process.env.PAPERCLIP_HOME, "config.json")); + } + candidates.push(path.join(home, ".paperclip", "instances", "default", "config.json")); + for (const candidate of candidates) { + if (fs.existsSync(candidate)) return candidate; + } + return null; +} + +function readJsonlSafe(filePath: string): string | null { + try { + return fs.readFileSync(filePath, "utf8"); + } catch (error) { + if (error && typeof error === "object" && "code" in error && (error as { code: string }).code === "ENOENT") { + return null; + } + throw error; + } +} + +interface Row { + id: string; + agentId: string; + taskKey: string; + sessionDisplayId: string | null; + sessionParamsJson: Record | null; +} + +function extractSessionParams(row: Row): ClaudeSessionRowInput { + const params = row.sessionParamsJson ?? {}; + const sessionId = + typeof params.sessionId === "string" && params.sessionId.length > 0 + ? (params.sessionId as string) + : null; + const cwd = + typeof params.cwd === "string" && params.cwd.length > 0 ? (params.cwd as string) : null; + const isRemote = + "remoteExecution" in params && + params.remoteExecution !== null && + typeof params.remoteExecution === "object"; + return { sessionId, cwd, isRemote }; +} + +interface HealedPair { + rowId: string; + agentId: string; + taskKey: string; + sessionId: string | null; + lastAssistantId: string | null; + jsonlPath: string | null; + reason: "poisoned" | "missing_jsonl"; +} + +async function main(argv: string[]): Promise { + const args = parseArgs(argv); + const configPath = args.configPath ?? defaultConfigPath(); + const databaseUrl = + args.databaseUrl ?? + process.env.DATABASE_URL ?? + (configPath ? readDatabaseUrlFromConfig(configPath) : null); + if (!databaseUrl) { + throw new Error( + "Unable to resolve database URL. Pass --database-url or --config .", + ); + } + const claudeConfigDir = resolveClaudeConfigDir(process.env, args.claudeConfigDir ?? undefined); + + const db = createDb(databaseUrl); + const closableDb = db as typeof db & { + $client?: { end?: (options?: { timeout?: number }) => Promise }; + }; + const summary = { + scanned: 0, + poisoned: 0, + missingJsonl: 0, + noAssistantEntries: 0, + healthy: 0, + skippedRemote: 0, + skippedNoSession: 0, + unreadable: 0, + deleted: 0, + dryRun: args.dryRun, + healedPairs: [] as HealedPair[], + skippedRemotePairs: [] as Array<{ rowId: string; agentId: string; taskKey: string }>, + }; + + const log = (line: string) => { + if (!args.json) process.stdout.write(`${line}\n`); + }; + + try { + const rows: Row[] = await db + .select({ + id: agentTaskSessions.id, + agentId: agentTaskSessions.agentId, + taskKey: agentTaskSessions.taskKey, + sessionDisplayId: agentTaskSessions.sessionDisplayId, + sessionParamsJson: agentTaskSessions.sessionParamsJson, + }) + .from(agentTaskSessions) + .where( + and( + eq(agentTaskSessions.adapterType, CLAUDE_ADAPTER_TYPE), + isNotNull(agentTaskSessions.sessionParamsJson), + ), + ); + + log( + `Scanning ${rows.length} claude_local rows; claudeConfigDir=${claudeConfigDir}; dryRun=${args.dryRun}`, + ); + + const toDelete: HealedPair[] = []; + for (const row of rows) { + summary.scanned += 1; + const input = extractSessionParams(row); + const classification = classifyRow(input, { + claudeConfigDir, + readJsonl: readJsonlSafe, + }); + switch (classification.kind) { + case "healthy": + summary.healthy += 1; + break; + case "no_assistant_entries": + summary.noAssistantEntries += 1; + break; + case "skipped_remote": + summary.skippedRemote += 1; + summary.skippedRemotePairs.push({ + rowId: row.id, + agentId: row.agentId, + taskKey: row.taskKey, + }); + break; + case "skipped_no_session_id": + summary.skippedNoSession += 1; + break; + case "unreadable_jsonl": + summary.unreadable += 1; + log( + `[warn] unreadable jsonl for row=${row.id} agent=${row.agentId} task=${row.taskKey} error=${classification.error}`, + ); + break; + case "missing_jsonl": { + summary.missingJsonl += 1; + if (args.deleteMissing) { + toDelete.push({ + rowId: row.id, + agentId: row.agentId, + taskKey: row.taskKey, + sessionId: input.sessionId, + lastAssistantId: null, + jsonlPath: classification.jsonlPath ?? null, + reason: "missing_jsonl", + }); + } else { + log( + `[skip] missing jsonl (use --delete-missing-jsonl to clear) row=${row.id} agent=${row.agentId} task=${row.taskKey} jsonl=${classification.jsonlPath}`, + ); + } + break; + } + case "poisoned": + summary.poisoned += 1; + toDelete.push({ + rowId: row.id, + agentId: row.agentId, + taskKey: row.taskKey, + sessionId: input.sessionId, + lastAssistantId: classification.lastAssistantId, + jsonlPath: classification.jsonlPath ?? null, + reason: "poisoned", + }); + break; + } + } + + if (toDelete.length === 0) { + log("No poisoned rows found."); + } else { + log(`Found ${toDelete.length} rows to clear:`); + for (const pair of toDelete) { + log( + ` - row=${pair.rowId} agent=${pair.agentId} task=${pair.taskKey} sessionId=${pair.sessionId} reason=${pair.reason} lastAssistantId=${pair.lastAssistantId ?? "n/a"}`, + ); + } + } + + if (!args.dryRun && toDelete.length > 0) { + // Single-statement delete with ids list; safer than a per-row loop and + // avoids races against concurrent heartbeat writes that might re-touch + // the same (agentId, taskKey). + const ids = toDelete.map((pair) => pair.rowId); + const result = await db + .delete(agentTaskSessions) + .where(sql`${agentTaskSessions.id} IN (${sql.join(ids.map((id) => sql`${id}`), sql`, `)})`) + .returning({ id: agentTaskSessions.id }); + summary.deleted = result.length; + log(`Deleted ${result.length} rows.`); + } + + summary.healedPairs = toDelete; + + if (args.json) { + process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`); + } else { + log("--- Summary ---"); + log( + `scanned=${summary.scanned} healthy=${summary.healthy} poisoned=${summary.poisoned} missingJsonl=${summary.missingJsonl} noAssistantEntries=${summary.noAssistantEntries} skippedRemote=${summary.skippedRemote} skippedNoSession=${summary.skippedNoSession} unreadable=${summary.unreadable} deleted=${summary.deleted} dryRun=${summary.dryRun}`, + ); + } + } finally { + await closableDb.$client?.end?.({ timeout: 5 }).catch(() => undefined); + } +} + +// Allow `import { ... } from "./clean-poisoned-claude-sessions.js"` from tests +// without triggering the CLI side-effect. Node sets `import.meta.main` on +// direct invocation; fall back to argv[1] comparison for older runtimes. +const invokedDirectly = + typeof process !== "undefined" && + Array.isArray(process.argv) && + process.argv[1] !== undefined && + path.resolve(process.argv[1]) === path.resolve(new URL(import.meta.url).pathname); + +if (invokedDirectly) { + main(process.argv.slice(2)).catch((error) => { + process.stderr.write( + `${error instanceof Error ? error.stack ?? error.message : String(error)}\n`, + ); + process.exit(1); + }); +} diff --git a/server/src/__tests__/claude-local-execute.test.ts b/server/src/__tests__/claude-local-execute.test.ts index 8cda4f0f..52527051 100644 --- a/server/src/__tests__/claude-local-execute.test.ts +++ b/server/src/__tests__/claude-local-execute.test.ts @@ -90,6 +90,72 @@ type CapturePayload = { appendedSystemPromptFileContents?: string | null; }; +async function writePoisonedMessageIdClaudeCommand(commandPath: string): Promise { + const script = `#!/usr/bin/env node +const fs = require("node:fs"); + +const capturePath = process.env.PAPERCLIP_TEST_CAPTURE_PATH; +const statePath = process.env.PAPERCLIP_TEST_STATE_PATH; +const payload = { + argv: process.argv.slice(2), + prompt: fs.readFileSync(0, "utf8"), +}; +if (capturePath) { + const entries = fs.existsSync(capturePath) ? JSON.parse(fs.readFileSync(capturePath, "utf8")) : []; + entries.push(payload); + fs.writeFileSync(capturePath, JSON.stringify(entries), "utf8"); +} +const resumed = process.argv.includes("--resume"); +const shouldFailResume = resumed && statePath && !fs.existsSync(statePath); +if (shouldFailResume) { + fs.writeFileSync(statePath, "retried", "utf8"); + console.log(JSON.stringify({ + type: "result", + subtype: "success", + session_id: "claude-session-poisoned", + is_error: true, + result: "API Error: 400 diagnostics.previous_message_id: must be the \`id\` from a prior /v1/messages response (starts with \`msg_\`)", + })); + process.exit(1); +} +console.log(JSON.stringify({ type: "system", subtype: "init", session_id: "claude-session-new", model: "claude-sonnet" })); +console.log(JSON.stringify({ type: "assistant", session_id: "claude-session-new", message: { content: [{ type: "text", text: "hello" }] } })); +console.log(JSON.stringify({ type: "result", session_id: "claude-session-new", result: "hello", usage: { input_tokens: 1, cache_read_input_tokens: 0, output_tokens: 1 } })); +`; + await fs.writeFile(commandPath, script, "utf8"); + await fs.chmod(commandPath, 0o755); +} + +async function writeAlwaysPoisonedMessageIdClaudeCommand(commandPath: string): Promise { + const script = `#!/usr/bin/env node +const fs = require("node:fs"); + +const capturePath = process.env.PAPERCLIP_TEST_CAPTURE_PATH; +const payload = { + argv: process.argv.slice(2), + prompt: fs.readFileSync(0, "utf8"), +}; +if (capturePath) { + const entries = fs.existsSync(capturePath) ? JSON.parse(fs.readFileSync(capturePath, "utf8")) : []; + entries.push(payload); + fs.writeFileSync(capturePath, JSON.stringify(entries), "utf8"); +} +// Both --resume and fresh attempts emit the poisoned previous_message_id result. +// The fresh attempt still carries a session_id in the result; the adapter must +// NOT persist it, otherwise the next continuation re-resumes a known-bad transcript. +console.log(JSON.stringify({ + type: "result", + subtype: "success", + session_id: "claude-session-poisoned-fresh", + is_error: true, + result: "API Error: 400 diagnostics.previous_message_id: must be the \`id\` from a prior /v1/messages response (starts with \`msg_\`)", +})); +process.exit(1); +`; + await fs.writeFile(commandPath, script, "utf8"); + await fs.chmod(commandPath, 0o755); +} + async function writeRetryThenSucceedClaudeCommand(commandPath: string): Promise { const script = `#!/usr/bin/env node const fs = require("node:fs"); @@ -1116,4 +1182,136 @@ describe("claude execute", () => { await fs.rm(root, { recursive: true, force: true }); } }); + + it("auto-rotates session on previous_message_id 400 (synthetic-msg poisoning) and succeeds on retry", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-exec-poisoned-msgid-")); + const { workspace, commandPath, capturePath, statePath, restore } = await setupExecuteEnv(root, { + commandWriter: writePoisonedMessageIdClaudeCommand, + }); + const logs: string[] = []; + try { + const result = await execute({ + runId: "run-poisoned-msgid", + agent: { id: "agent-1", companyId: "co-1", name: "Test", adapterType: "claude_local", adapterConfig: {} }, + runtime: { sessionId: "claude-session-poisoned", sessionParams: null, sessionDisplayId: null, taskKey: null }, + config: { + command: commandPath, + cwd: workspace, + env: { + PAPERCLIP_TEST_CAPTURE_PATH: capturePath, + PAPERCLIP_TEST_STATE_PATH: statePath, + }, + promptTemplate: "Do work.", + }, + context: {}, + authToken: "tok", + onLog: async (_stream, chunk) => { logs.push(chunk); }, + }); + + const captured: Array<{ argv: string[] }> = JSON.parse(await fs.readFile(capturePath, "utf-8")); + // First attempt resumes, second attempt starts fresh + expect(captured).toHaveLength(2); + expect(captured[0]?.argv).toContain("--resume"); + expect(captured[1]?.argv).not.toContain("--resume"); + // Result comes from the fresh retry + expect(result.sessionId).toBe("claude-session-new"); + expect(result.errorCode).toBeNull(); + // Adapter logged the fallback reason + expect(logs.some((l) => l.includes("poisoned message-id"))).toBe(true); + } finally { + restore(); + await fs.rm(root, { recursive: true, force: true }); + } + }); + + /** + * Regression for RED-978: the adapter must not persist a sessionId from a + * run that ended with a poisoned previous_message_id error. Otherwise the + * next continuation auto-resumes a known-bad transcript and Anthropic + * /v1/messages returns 400 again, permanently stranding the issue. + */ + it("drops sessionId and forces clearSession when a fresh run reports a poisoned previous_message_id", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-exec-poisoned-fresh-")); + const { workspace, commandPath, capturePath, restore } = await setupExecuteEnv(root, { + commandWriter: writeAlwaysPoisonedMessageIdClaudeCommand, + }); + try { + const result = await execute({ + runId: "run-poisoned-fresh", + agent: { id: "agent-1", companyId: "co-1", name: "Test", adapterType: "claude_local", adapterConfig: {} }, + runtime: { sessionId: null, sessionParams: null, sessionDisplayId: null, taskKey: null }, + config: { + command: commandPath, + cwd: workspace, + env: { PAPERCLIP_TEST_CAPTURE_PATH: capturePath }, + promptTemplate: "Do work.", + }, + context: {}, + authToken: "tok", + onLog: async () => {}, + }); + + // The fake CLI emits a session_id in its poisoned result; the adapter + // must not propagate it. The server uses clearSession=true to wipe + // any previously-persisted session state for this issue/task. + expect(result.sessionId).toBeNull(); + expect(result.sessionParams).toBeNull(); + expect(result.sessionDisplayId).toBeNull(); + expect(result.clearSession).toBe(true); + expect(result.errorCode).toBe("claude_poisoned_previous_message_id"); + expect(result.errorMessage ?? "").toContain("previous_message_id"); + } finally { + restore(); + await fs.rm(root, { recursive: true, force: true }); + } + }); + + /** + * Regression for RED-978: if the auto-retry after a poisoned resume *also* + * fails with a poisoned previous_message_id, the adapter must still emit + * clearSession=true so the next heartbeat starts from a clean transcript. + * Before this fix, the retry result's session_id ("claude-session-poisoned-fresh") + * was persisted and every subsequent continuation hit the same 400 again. + */ + it("forces clearSession when the recovery retry also reports a poisoned previous_message_id", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-exec-poisoned-retry-")); + const { workspace, commandPath, capturePath, restore } = await setupExecuteEnv(root, { + commandWriter: writeAlwaysPoisonedMessageIdClaudeCommand, + }); + try { + const result = await execute({ + runId: "run-poisoned-retry", + agent: { id: "agent-1", companyId: "co-1", name: "Test", adapterType: "claude_local", adapterConfig: {} }, + runtime: { + sessionId: "claude-session-already-poisoned", + sessionParams: null, + sessionDisplayId: null, + taskKey: null, + }, + config: { + command: commandPath, + cwd: workspace, + env: { PAPERCLIP_TEST_CAPTURE_PATH: capturePath }, + promptTemplate: "Do work.", + }, + context: {}, + authToken: "tok", + onLog: async () => {}, + }); + + const captured: Array<{ argv: string[] }> = JSON.parse(await fs.readFile(capturePath, "utf-8")); + // Resume attempt + fresh recovery attempt, both poisoned. + expect(captured).toHaveLength(2); + expect(captured[0]?.argv).toContain("--resume"); + expect(captured[1]?.argv).not.toContain("--resume"); + // Crucially: do NOT persist the retry's reported sessionId. + expect(result.sessionId).toBeNull(); + expect(result.sessionParams).toBeNull(); + expect(result.clearSession).toBe(true); + expect(result.errorCode).toBe("claude_poisoned_previous_message_id"); + } finally { + restore(); + await fs.rm(root, { recursive: true, force: true }); + } + }); });