import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { activityLog, agents, companies, createDb, heartbeatRuns, issueComments, issueRelations, issues, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; const mockTelemetryClient = vi.hoisted(() => ({ track: vi.fn() })); vi.mock("../telemetry.ts", () => ({ getTelemetryClient: () => mockTelemetryClient })); import { heartbeatService } from "../services/heartbeat.ts"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { console.warn( `Skipping embedded Postgres stale-lock sweeper tests on this host: ${ embeddedPostgresSupport.reason ?? "unsupported environment" }`, ); } describeEmbeddedPostgres("recovery sweepStaleIssueLocks", () => { let db!: ReturnType; let tempDb: Awaited> | null = null; beforeAll(async () => { tempDb = await startEmbeddedPostgresTestDatabase("paperclip-stale-lock-sweep-"); db = createDb(tempDb.connectionString); }, 20_000); afterEach(async () => { await db.delete(issueComments); await db.delete(issueRelations); await db.delete(activityLog); await db.delete(issues); await db.delete(heartbeatRuns); await db.delete(agents); await db.delete(companies); }); afterAll(async () => { await tempDb?.cleanup(); }); async function seed() { const companyId = randomUUID(); const agentId = randomUUID(); const failedRunId = randomUUID(); const runningRunId = randomUUID(); await db.insert(companies).values({ id: companyId, name: "Paperclip", issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, requireBoardApprovalForNewAgents: false, }); await db.insert(agents).values({ id: agentId, companyId, name: "Coder", role: "engineer", status: "active", adapterType: "codex_local", adapterConfig: {}, runtimeConfig: {}, permissions: {}, }); await db.insert(heartbeatRuns).values([ { id: failedRunId, companyId, agentId, status: "failed", invocationSource: "manual", finishedAt: new Date(), }, { id: runningRunId, companyId, agentId, status: "running", invocationSource: "manual", startedAt: new Date(), }, ]); return { companyId, agentId, failedRunId, runningRunId }; } it("clears lock columns when checkoutRunId points at a terminal heartbeat run", async () => { const { companyId, agentId, failedRunId } = await seed(); const issueId = randomUUID(); await db.insert(issues).values({ id: issueId, companyId, title: "Stale lock — terminal checkoutRunId", // Status off in_progress + checkoutRunId still set → exactly the recurrence shape. status: "todo", priority: "high", assigneeAgentId: agentId, checkoutRunId: failedRunId, executionRunId: null, }); const heartbeat = heartbeatService(db); const result = await heartbeat.sweepStaleIssueLocks(); expect(result.cleared).toBe(1); expect(result.issueIds).toEqual([issueId]); const row = await db .select({ checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId, executionLockedAt: issues.executionLockedAt, }) .from(issues) .where(eq(issues.id, issueId)) .then((rows) => rows[0]); expect(row).toEqual({ checkoutRunId: null, executionRunId: null, executionLockedAt: null }); const audit = await db .select({ action: activityLog.action, details: activityLog.details }) .from(activityLog) .where(eq(activityLog.action, "issue.stale_lock_cleared")) .then((rows) => rows[0]); expect(audit?.action).toBe("issue.stale_lock_cleared"); expect((audit?.details as { clearedCheckoutRunId?: string } | null)?.clearedCheckoutRunId).toBe( failedRunId, ); }); it("does not clear locks while the referenced run is still running", async () => { const { companyId, agentId, runningRunId } = await seed(); const issueId = randomUUID(); await db.insert(issues).values({ id: issueId, companyId, title: "Live lock — must be preserved", status: "in_progress", priority: "high", assigneeAgentId: agentId, checkoutRunId: runningRunId, executionRunId: runningRunId, executionLockedAt: new Date(), }); const heartbeat = heartbeatService(db); const result = await heartbeat.sweepStaleIssueLocks(); expect(result.cleared).toBe(0); const row = await db .select({ checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId, }) .from(issues) .where(eq(issues.id, issueId)) .then((rows) => rows[0]); expect(row).toEqual({ checkoutRunId: runningRunId, executionRunId: runningRunId }); }); it("does not clear when checkoutRunId is terminal but executionRunId is still running", async () => { const { companyId, agentId, failedRunId, runningRunId } = await seed(); const issueId = randomUUID(); await db.insert(issues).values({ id: issueId, companyId, title: "Mixed lock — preserve", status: "in_progress", priority: "high", assigneeAgentId: agentId, checkoutRunId: failedRunId, executionRunId: runningRunId, executionLockedAt: new Date(), }); const heartbeat = heartbeatService(db); const result = await heartbeat.sweepStaleIssueLocks(); expect(result.cleared).toBe(0); const row = await db .select({ checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId, }) .from(issues) .where(eq(issues.id, issueId)) .then((rows) => rows[0]); expect(row).toEqual({ checkoutRunId: failedRunId, executionRunId: runningRunId }); }); it("is idempotent — second pass finds nothing to clear", async () => { const { companyId, agentId, failedRunId } = await seed(); const issueId = randomUUID(); await db.insert(issues).values({ id: issueId, companyId, title: "Idempotency", status: "todo", priority: "high", assigneeAgentId: agentId, checkoutRunId: failedRunId, executionRunId: null, }); const heartbeat = heartbeatService(db); const first = await heartbeat.sweepStaleIssueLocks(); const second = await heartbeat.sweepStaleIssueLocks(); expect(first.cleared).toBe(1); expect(second.cleared).toBe(0); }); });