fix(codex-local): seed managed auth into isolated homes (#8403)

## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The `codex_local` adapter isolates local Codex runs by assigning managed `CODEX_HOME` state per company and per agent > - PR #8272 tightened that isolation, but it left a gap: once `CODEX_HOME` became explicit, the adapter treated it like a user-managed override and skipped auth seeding > - That meant newly isolated agents could launch with no usable `auth.json`, hit OpenAI unauthenticated, and fail with `401 Missing bearer` > - Users who had already persisted one of those broken managed homes could remain stranded even after config changes unless Paperclip repaired the home itself > - This pull request teaches Paperclip to seed managed homes correctly, backfill already-stranded managed homes on startup, and reject credential-less managed homes before they reach the provider > - The benefit is that affected managed `codex_local` agents recover automatically after upgrade and restart, without manual `CODEX_HOME` surgery ## Linked Issues or Issue Description - Fixes #497 - Refs #5028 - Related PR: #8272 - Related PR: #8399 ## What Changed - Distinguished Paperclip-managed `CODEX_HOME` paths from genuine external overrides and always seeded auth into managed homes, even when `CODEX_HOME` is explicit in config. - Wrote API-key-backed `auth.json` files for managed homes when `OPENAI_API_KEY` is configured, otherwise symlinked the shared Codex auth for subscription/OAuth flows. - Added a startup reconciliation pass that backfills already-isolated managed homes created by the broken release so upgrade plus restart repairs stranded agents automatically. - Preserved previously resolved API-key auth when the stored `OPENAI_API_KEY` binding is secret-backed and startup cannot resolve the secret value directly. - Hardened the managed-home preflight to require a credential-bearing `auth.json`, not just file presence, and documented the recovery behavior. - Added regression tests covering managed-home seeding, fail-fast behavior, and server-side startup reconciliation. ## Verification ```bash pnpm exec vitest run packages/adapters/codex-local/src/server/codex-home.test.ts packages/adapters/codex-local/src/server/execute.auth.test.ts server/src/__tests__/codex-auth-reconciliation.test.ts server/src/__tests__/codex-local-execute.test.ts server/src/__tests__/server-startup-feedback-export.test.ts pnpm --filter @paperclipai/adapter-codex-local typecheck pnpm --filter @paperclipai/server typecheck pnpm check:tokens pnpm exec vitest run server/src/__tests__/server-startup-feedback-export.test.ts ``` GitHub Actions `PR` workflow is green on latest head `e5b1e08d4`, including policy, typecheck, test shards, build, e2e, serialized server suites, and canary dry run. Greptile Review is green on latest head with 0 comments added. No UI changes. ## Risks - Startup reconciliation now mutates persisted managed Codex homes at boot. Risk is low because it only touches Paperclip-managed company/agent home paths and no-ops when a home already has usable auth. - Genuine external `CODEX_HOME` overrides remain intentionally self-managed, so those users still own repair steps inside their custom home. - Hosts with neither shared Codex auth nor an explicit per-agent API key now fail earlier with a clearer adapter error instead of surfacing a downstream `401`, which changes timing but not capability. ## Model Used - OpenAI Codex, GPT-5-based coding agent in a local Codex session; exact served model ID/context window were not exposed to the session. Tool use and code execution were enabled. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
2026-06-20 16:17:54 -07:00
parent e281095cb4
commit 67ca8287e8
12 changed files with 1034 additions and 14 deletions
@@ -0,0 +1,206 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import type { Db } from "@paperclipai/db";
+import { reconcileCodexLocalManagedHomesOnStartup } from "../services/codex-auth-reconciliation.js";
+
+type AgentRow = {
+  id: string;
+  companyId: string;
+  adapterConfig: Record<string, unknown>;
+};
+
+function makeDb(rows: AgentRow[]): Db {
+  return {
+    select: () => ({
+      from: () => ({
+        where: async () => rows,
+      }),
+    }),
+  } as unknown as Db;
+}
+
+describe("reconcileCodexLocalManagedHomesOnStartup", () => {
+  let root: string;
+  let paperclipHome: string;
+  let sharedCodexHome: string;
+  const savedEnv: Record<string, string | undefined> = {};
+
+  function managedAgentHome(companyId: string, agentId: string): string {
+    return path.join(
+      paperclipHome,
+      "instances",
+      "default",
+      "companies",
+      companyId,
+      "agents",
+      agentId,
+      "codex-home",
+    );
+  }
+
+  beforeEach(async () => {
+    root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-startup-"));
+    paperclipHome = path.join(root, "paperclip-home");
+    sharedCodexHome = path.join(root, "shared-codex-home");
+    await fs.mkdir(sharedCodexHome, { recursive: true });
+    await fs.writeFile(path.join(sharedCodexHome, "auth.json"), '{"token":"shared"}', "utf8");
+
+    for (const key of ["PAPERCLIP_HOME", "PAPERCLIP_INSTANCE_ID", "CODEX_HOME"]) {
+      savedEnv[key] = process.env[key];
+    }
+    process.env.PAPERCLIP_HOME = paperclipHome;
+    process.env.PAPERCLIP_INSTANCE_ID = "default";
+    process.env.CODEX_HOME = sharedCodexHome;
+  });
+
+  afterEach(async () => {
+    for (const [key, value] of Object.entries(savedEnv)) {
+      if (value === undefined) delete process.env[key];
+      else process.env[key] = value;
+    }
+    await fs.rm(root, { recursive: true, force: true });
+  });
+
+  it("seeds a stranded managed home and is a no-op on the next boot", async () => {
+    const agentHome = managedAgentHome("company-1", "agent-7");
+    const rows: AgentRow[] = [
+      { id: "agent-7", companyId: "company-1", adapterConfig: { env: { CODEX_HOME: agentHome, OPENAI_API_KEY: "" } } },
+    ];
+
+    const first = await reconcileCodexLocalManagedHomesOnStartup(makeDb(rows));
+    expect(first).toMatchObject({ scanned: 1, seeded: 1, alreadySeeded: 0, failed: 0 });
+    expect(first.seededAgentIds).toEqual(["agent-7"]);
+    const agentAuth = path.join(agentHome, "auth.json");
+    expect((await fs.lstat(agentAuth)).isSymbolicLink()).toBe(true);
+    expect(await fs.realpath(agentAuth)).toBe(
+      await fs.realpath(path.join(sharedCodexHome, "auth.json")),
+    );
+
+    const second = await reconcileCodexLocalManagedHomesOnStartup(makeDb(rows));
+    expect(second).toMatchObject({ scanned: 1, seeded: 0, alreadySeeded: 1, failed: 0 });
+  });
+
+  it("does not report a backfill when shared Codex auth is missing", async () => {
+    await fs.rm(path.join(sharedCodexHome, "auth.json"), { force: true });
+    const agentHome = managedAgentHome("company-1", "agent-missing-source");
+    const rows: AgentRow[] = [
+      { id: "agent-missing-source", companyId: "company-1", adapterConfig: { env: { CODEX_HOME: agentHome } } },
+    ];
+
+    const summary = await reconcileCodexLocalManagedHomesOnStartup(makeDb(rows));
+    expect(summary).toMatchObject({
+      scanned: 1,
+      seeded: 0,
+      sourceAuthMissing: 1,
+      failed: 0,
+    });
+    await expect(fs.lstat(path.join(agentHome, "auth.json"))).rejects.toThrow();
+  });
+
+  it("classifies external overrides and unconfigured homes without seeding", async () => {
+    const external = path.join(root, "user-codex");
+    await fs.mkdir(external, { recursive: true });
+    const rows: AgentRow[] = [
+      { id: "ext", companyId: "company-1", adapterConfig: { env: { CODEX_HOME: external } } },
+      { id: "none", companyId: "company-1", adapterConfig: { env: {} } },
+    ];
+
+    const summary = await reconcileCodexLocalManagedHomesOnStartup(makeDb(rows));
+    expect(summary).toMatchObject({
+      scanned: 2,
+      seeded: 0,
+      externalOverride: 1,
+      noManagedHome: 1,
+      failed: 0,
+    });
+    expect(await fs.access(path.join(external, "auth.json")).then(() => true).catch(() => false)).toBe(
+      false,
+    );
+  });
+
+  it("does not write a secret-bound OPENAI_API_KEY placeholder as the API key", async () => {
+    const agentHome = managedAgentHome("company-2", "agent-9");
+    // A secret binding must never be written verbatim into auth.json; the home
+    // should fall back to the seeded subscription symlink instead.
+    const rows: AgentRow[] = [
+      {
+        id: "agent-9",
+        companyId: "company-2",
+        adapterConfig: {
+          env: {
+            CODEX_HOME: agentHome,
+            OPENAI_API_KEY: { type: "secret", secretId: "sec-123" },
+          },
+        },
+      },
+    ];
+
+    const summary = await reconcileCodexLocalManagedHomesOnStartup(makeDb(rows));
+    expect(summary).toMatchObject({ scanned: 1, seeded: 1, failed: 0 });
+    const agentAuth = path.join(agentHome, "auth.json");
+    expect((await fs.lstat(agentAuth)).isSymbolicLink()).toBe(true);
+    expect(await fs.realpath(agentAuth)).toBe(
+      await fs.realpath(path.join(sharedCodexHome, "auth.json")),
+    );
+  });
+
+  it("preserves a preexisting API-key auth.json when the key is secret-bound", async () => {
+    const agentHome = managedAgentHome("company-4", "agent-secret");
+    // Simulate an agent that previously ran with a RESOLVED secret API key:
+    // execute-time seeding wrote a regular-file auth.json containing the key.
+    await fs.mkdir(agentHome, { recursive: true });
+    await fs.writeFile(
+      path.join(agentHome, "auth.json"),
+      JSON.stringify({ OPENAI_API_KEY: "sk-secret-resolved" }),
+      { mode: 0o600 },
+    );
+    const rows: AgentRow[] = [
+      {
+        id: "agent-secret",
+        companyId: "company-4",
+        adapterConfig: {
+          env: {
+            CODEX_HOME: agentHome,
+            // Canonical secret binding shape; unresolvable at startup.
+            OPENAI_API_KEY: { type: "secret_ref", secretId: "11111111-1111-1111-1111-111111111111" },
+          },
+        },
+      },
+    ];
+
+    const summary = await reconcileCodexLocalManagedHomesOnStartup(makeDb(rows));
+    expect(summary).toMatchObject({ scanned: 1, seeded: 0, alreadySeeded: 1, failed: 0 });
+
+    const agentAuth = path.join(agentHome, "auth.json");
+    // The resolved-key file must remain a regular file, not be downgraded to the
+    // shared subscription symlink.
+    expect((await fs.lstat(agentAuth)).isSymbolicLink()).toBe(false);
+    expect(JSON.parse(await fs.readFile(agentAuth, "utf8"))).toEqual({
+      OPENAI_API_KEY: "sk-secret-resolved",
+    });
+  });
+
+  it("writes a plain OPENAI_API_KEY into the managed home", async () => {
+    const agentHome = managedAgentHome("company-3", "agent-5");
+    const rows: AgentRow[] = [
+      {
+        id: "agent-5",
+        companyId: "company-3",
+        adapterConfig: { env: { CODEX_HOME: agentHome, OPENAI_API_KEY: "sk-plain-1" } },
+      },
+    ];
+
+    const summary = await reconcileCodexLocalManagedHomesOnStartup(makeDb(rows));
+    expect(summary).toMatchObject({ scanned: 1, seeded: 1, failed: 0 });
+    const written = JSON.parse(await fs.readFile(path.join(agentHome, "auth.json"), "utf8"));
+    expect(written).toEqual({ OPENAI_API_KEY: "sk-plain-1" });
+
+    const second = await reconcileCodexLocalManagedHomesOnStartup(makeDb(rows));
+    expect(second).toMatchObject({ scanned: 1, seeded: 0, alreadySeeded: 1, failed: 0 });
+    expect(JSON.parse(await fs.readFile(path.join(agentHome, "auth.json"), "utf8"))).toEqual({
+      OPENAI_API_KEY: "sk-plain-1",
+    });
+  });
+});
@@ -58,6 +58,12 @@ type LogEntry = {
  chunk: string;
 };

+async function seedSharedCodexAuth(homeRoot: string): Promise<void> {
+  const sharedCodexHome = path.join(homeRoot, ".codex");
+  await fs.mkdir(sharedCodexHome, { recursive: true });
+  await fs.writeFile(path.join(sharedCodexHome, "auth.json"), '{"token":"shared"}\n', "utf8");
+}
+
 function createLocalSandboxRunner() {
  let counter = 0;
  return {
@@ -201,6 +207,7 @@ describe("codex execute", () => {

    const previousHome = process.env.HOME;
    process.env.HOME = root;
+    await seedSharedCodexAuth(root);

    let commandNotes: string[] = [];
    try {
@@ -261,6 +268,7 @@ describe("codex execute", () => {
    const previousPath = process.env.PATH;
    process.env.HOME = root;
    process.env.PATH = `${binDir}${path.delimiter}${process.env.PATH ?? ""}`;
+    await seedSharedCodexAuth(root);

    let loggedCommand: string | null = null;
    let loggedEnv: Record<string, string> = {};
@@ -328,6 +336,7 @@ describe("codex execute", () => {

    process.env.HOME = root;
    process.env.PATH = `${binDir}${path.delimiter}${process.env.PATH ?? ""}`;
+    await seedSharedCodexAuth(root);

    try {
      const result = await execute({
@@ -395,6 +404,7 @@ describe("codex execute", () => {

    const previousHome = process.env.HOME;
    process.env.HOME = root;
+    await seedSharedCodexAuth(root);

    try {
      const result = await execute({
@@ -505,6 +515,7 @@ describe("codex execute", () => {

    const previousHome = process.env.HOME;
    process.env.HOME = root;
+    await seedSharedCodexAuth(root);

    try {
      const result = await execute({
@@ -555,6 +566,7 @@ describe("codex execute", () => {

    const previousHome = process.env.HOME;
    process.env.HOME = root;
+    await seedSharedCodexAuth(root);
    vi.useFakeTimers();
    vi.setSystemTime(new Date(2026, 3, 22, 22, 29, 0));

@@ -615,6 +627,7 @@ describe("codex execute", () => {

    const previousHome = process.env.HOME;
    process.env.HOME = root;
+    await seedSharedCodexAuth(root);

    let commandNotes: string[] = [];
    try {
@@ -691,6 +704,7 @@ describe("codex execute", () => {

    const previousHome = process.env.HOME;
    process.env.HOME = root;
+    await seedSharedCodexAuth(root);

    try {
      const result = await execute({
@@ -845,6 +859,7 @@ describe("codex execute", () => {

    const previousHome = process.env.HOME;
    process.env.HOME = root;
+    await seedSharedCodexAuth(root);

    try {
      const result = await execute({
@@ -943,6 +958,7 @@ describe("codex execute", () => {

    const previousHome = process.env.HOME;
    process.env.HOME = root;
+    await seedSharedCodexAuth(root);

    let invocationPrompt = "";
    let invocationNotes: string[] = [];
@@ -168,6 +168,16 @@ vi.mock("../services/index.js", () => ({
    })),
  })),
  reconcileCloudUpstreamRunsOnStartup: vi.fn(async () => ({ reconciled: 0 })),
+  reconcileCodexLocalManagedHomesOnStartup: vi.fn(async () => ({
+    scanned: 0,
+    seeded: 0,
+    alreadySeeded: 0,
+    externalOverride: 0,
+    noManagedHome: 0,
+    sourceAuthMissing: 0,
+    failed: 0,
+    seededAgentIds: [],
+  })),
  reconcilePersistedRuntimeServicesOnStartup: vi.fn(async () => ({ reconciled: 0 })),
  routineService: vi.fn(() => ({
    tickScheduledTriggers: vi.fn(async () => ({ triggered: 0 })),
@@ -41,6 +41,7 @@ import {
  heartbeatService,
  instanceSettingsService,
  reconcileCloudUpstreamRunsOnStartup,
+  reconcileCodexLocalManagedHomesOnStartup,
  reconcilePersistedRuntimeServicesOnStartup,
  routineService,
 } from "./services/index.js";
@@ -728,7 +729,29 @@ export async function startServer(): Promise<StartedServer> {
    .catch((err) => {
      logger.error({ err }, "startup reconciliation of cloud upstream runs failed");
    });
-  
+
+  // Backfill auth.json into any already-isolated codex_local managed home that
+  // was created by the #8272 isolation guard before the Phase 1 seeding fix.
+  // Idempotent; the Phase 1 execute-time seeding covers new strandings.
+  void reconcileCodexLocalManagedHomesOnStartup(db)
+    .then((result) => {
+      if (result.seeded > 0 || result.failed > 0) {
+        logger.warn(
+          { seeded: result.seeded, failed: result.failed, scanned: result.scanned },
+          "reconciled codex_local managed homes (backfilled missing auth)",
+        );
+      }
+      if (result.sourceAuthMissing > 0) {
+        logger.warn(
+          { sourceAuthMissing: result.sourceAuthMissing, scanned: result.scanned },
+          "could not backfill codex_local managed homes because shared Codex auth is missing",
+        );
+      }
+    })
+    .catch((err) => {
+      logger.error({ err }, "startup reconciliation of codex_local managed homes failed");
+    });
+
  // Force the instance onto the Kubernetes sandbox provider when configured via
  // env (PAPERCLIP_EXECUTION_MODE=kubernetes). Runs BEFORE the heartbeat resumes
  // queued runs so the policy + managed k8s environments are in place. A bad
@@ -0,0 +1,147 @@
+import type { Db } from "@paperclipai/db";
+import { agents } from "@paperclipai/db";
+import { reconcileManagedCodexHome } from "@paperclipai/adapter-codex-local/server";
+import { eq } from "drizzle-orm";
+import { logger } from "../middleware/logger.js";
+
+export interface CodexAuthReconciliationSummary {
+  scanned: number;
+  seeded: number;
+  alreadySeeded: number;
+  externalOverride: number;
+  noManagedHome: number;
+  sourceAuthMissing: number;
+  failed: number;
+  seededAgentIds: string[];
+}
+
+function asRecord(value: unknown): Record<string, unknown> | null {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : null;
+}
+
+/**
+ * Extracts a literal (non-secret) env value. Adapter env bindings persist either
+ * as a bare string or as a `{ type: "plain", value }` object; secret bindings
+ * are intentionally NOT resolved here (we never write an unresolved secret
+ * placeholder into auth.json). Mirrors the server-side env-binding extraction in
+ * routes/agents.ts.
+ */
+function readPlainEnvValue(value: unknown): string | null {
+  if (typeof value === "string") {
+    const trimmed = value.trim();
+    return trimmed.length > 0 ? trimmed : null;
+  }
+  const record = asRecord(value);
+  if (record?.type !== "plain") return null;
+  return readPlainEnvValue(record.value);
+}
+
+type ApiKeyBinding =
+  | { kind: "plain"; value: string }
+  | { kind: "secret" }
+  | { kind: "none" };
+
+/**
+ * Classifies an `OPENAI_API_KEY` env binding so reconciliation can tell the
+ * difference between three states it must treat differently:
+ *  - `plain`: a literal value we can write into auth.json.
+ *  - `secret`: a secret binding (e.g. `{ type: "secret_ref", ... }`) we cannot
+ *    resolve at startup; the resolved value may already exist on disk from a
+ *    prior execute-time run, so reconciliation must not clobber it.
+ *  - `none`: no key configured (chatgpt-subscription mode); seed the shared
+ *    auth symlink.
+ */
+function classifyApiKeyBinding(value: unknown): ApiKeyBinding {
+  const plain = readPlainEnvValue(value);
+  if (plain) return { kind: "plain", value: plain };
+  const record = asRecord(value);
+  if (record && typeof record.type === "string" && record.type !== "plain") {
+    return { kind: "secret" };
+  }
+  return { kind: "none" };
+}
+
+/**
+ * Startup backfill: seed `auth.json` into any already-
+ * isolated `codex_local` managed home that was created (by the #8272 isolation
+ * guard) before the Phase 1 seeding fix landed. Phase 1 seeds at execute time;
+ * this repairs persisted homes proactively so a stranded agent recovers without
+ * waiting to run and without a manual symlink. Idempotent and safe to re-run on
+ * every boot: a home that already has valid auth is a no-op.
+ */
+export async function reconcileCodexLocalManagedHomesOnStartup(
+  db: Db,
+): Promise<CodexAuthReconciliationSummary> {
+  const summary: CodexAuthReconciliationSummary = {
+    scanned: 0,
+    seeded: 0,
+    alreadySeeded: 0,
+    externalOverride: 0,
+    noManagedHome: 0,
+    sourceAuthMissing: 0,
+    failed: 0,
+    seededAgentIds: [],
+  };
+
+  const rows = await db
+    .select({
+      id: agents.id,
+      companyId: agents.companyId,
+      adapterConfig: agents.adapterConfig,
+    })
+    .from(agents)
+    .where(eq(agents.adapterType, "codex_local"));
+
+  for (const row of rows) {
+    summary.scanned += 1;
+    const env = asRecord(asRecord(row.adapterConfig)?.env);
+    const configuredCodexHome = env ? readPlainEnvValue(env.CODEX_HOME) : null;
+    const apiKeyBinding = classifyApiKeyBinding(env?.OPENAI_API_KEY);
+
+    try {
+      const result = await reconcileManagedCodexHome({
+        companyId: row.companyId,
+        configuredCodexHome,
+        apiKey: apiKeyBinding.kind === "plain" ? apiKeyBinding.value : null,
+        apiKeySecretBound: apiKeyBinding.kind === "secret",
+      });
+      switch (result.status) {
+        case "seeded":
+          summary.seeded += 1;
+          summary.seededAgentIds.push(row.id);
+          logger.info(
+            { agentId: row.id, companyId: row.companyId, home: result.home },
+            "seeded auth into already-isolated codex_local managed home",
+          );
+          break;
+        case "already_seeded":
+          summary.alreadySeeded += 1;
+          break;
+        case "external_override":
+          summary.externalOverride += 1;
+          break;
+        case "no_managed_home":
+          summary.noManagedHome += 1;
+          break;
+        case "source_auth_missing":
+          summary.sourceAuthMissing += 1;
+          break;
+      }
+    } catch (err) {
+      summary.failed += 1;
+      logger.warn(
+        {
+          agentId: row.id,
+          companyId: row.companyId,
+          home: configuredCodexHome,
+          err: err instanceof Error ? err.message : String(err),
+        },
+        "failed to reconcile codex_local managed home on startup",
+      );
+    }
+  }
+
+  return summary;
+}
@@ -80,5 +80,9 @@ export { workProductService } from "./work-products.js";
 export { logActivity, type LogActivityInput } from "./activity-log.js";
 export { notifyHireApproved, type NotifyHireApprovedInput } from "./hire-hook.js";
 export { publishLiveEvent, subscribeCompanyLiveEvents } from "./live-events.js";
+export {
+  reconcileCodexLocalManagedHomesOnStartup,
+  type CodexAuthReconciliationSummary,
+} from "./codex-auth-reconciliation.js";
 export { reconcilePersistedRuntimeServicesOnStartup, restartDesiredRuntimeServicesOnStartup } from "./workspace-runtime.js";
 export { createStorageServiceFromConfig, getStorageService } from "../storage/index.js";