diff --git a/packages/plugins/sdk/src/protocol.ts b/packages/plugins/sdk/src/protocol.ts index 92fcbf42..5607b470 100644 --- a/packages/plugins/sdk/src/protocol.ts +++ b/packages/plugins/sdk/src/protocol.ts @@ -478,6 +478,13 @@ export interface PluginEnvironmentAcquireLeaseParams extends PluginEnvironmentDr runId: string; workspaceMode?: string; requestedCwd?: string; + /** + * The harness/adapter type for THIS run (the agent's adapter), so a single + * environment can serve mixed harnesses. When omitted, the driver falls back to + * the environment's configured default adapter. A provider that materializes a + * per-run sandbox should use this to select the runtime image and per-run env. + */ + adapterType?: string; } export interface PluginEnvironmentResumeLeaseParams extends PluginEnvironmentDriverBaseParams { diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 74b55c61..0a1d3f79 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -393,6 +393,7 @@ export type { AgentSkillEntry, AgentSkillSnapshot, AgentSkillSyncRequest, + InstanceExecutionMode, InstanceExperimentalSettings, InstanceGeneralSettings, InstanceSettings, @@ -1400,3 +1401,11 @@ export type { EnvironmentProviderCapability, EnvironmentSupportStatus, } from "./environment-support.js"; + +export type { AdapterRegistryEntry } from "./types/adapter-registry.js"; + +export { + adapterRegistryEntrySchema, + adapterRegistrySchema, + type AdapterRegistryEntryParsed, +} from "./validators/adapter-registry.js"; diff --git a/packages/shared/src/types/adapter-registry.ts b/packages/shared/src/types/adapter-registry.ts new file mode 100644 index 00000000..bd9b0a1f --- /dev/null +++ b/packages/shared/src/types/adapter-registry.ts @@ -0,0 +1,30 @@ +/** + * One declarative agent-harness ("adapter") entry. The same shape is used for + * local self-hosting and our operator/cloud: it governs both availability (the + * picker) and, when the run is sandboxed on Kubernetes, the runtime wiring. + * + * Replace semantics: when a registry is supplied it is the COMPLETE declared + * set. Adopt (built-in defaults) = no registry at all. Remove = omit the entry. + * Add = include a new entry. Override = redefine an existing adapterType. + */ +export interface AdapterRegistryEntry { + /** The harness, e.g. "opencode_local". */ + adapterType: string; + /** Availability (both local + k8s). Default true. */ + enabled?: boolean; + /** k8s-sandbox-only: container image the Job/Sandbox runs. */ + runtimeImage?: string; + /** k8s-sandbox-only: process-env keys forwarded into the Job (e.g. ANTHROPIC_API_KEY). */ + envKeys?: string[]; + /** k8s-sandbox-only: egress FQDN allow-list for the agent pod. */ + allowFqdns?: string[]; + /** k8s-sandbox-only: liveness/probe command. */ + probeCommand?: string[]; + /** + * Non-secret env injected into the Job/Sandbox as the BASE; the process-env + * values (the secret API key, via envKeys) override it. Carries e.g. + * ANTHROPIC_BASE_URL pointing at the in-cluster Bifrost gateway. NEVER put + * secrets here. + */ + defaultEnv?: Record; +} diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index fead54e4..5fc8b5f8 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -24,6 +24,7 @@ export type { FeedbackTraceBundle, } from "./feedback.js"; export type { + InstanceExecutionMode, InstanceExperimentalSettings, InstanceGeneralSettings, InstanceSettings, diff --git a/packages/shared/src/types/instance.ts b/packages/shared/src/types/instance.ts index 0a6c946c..05e6270a 100644 --- a/packages/shared/src/types/instance.ts +++ b/packages/shared/src/types/instance.ts @@ -19,11 +19,29 @@ export const DEFAULT_BACKUP_RETENTION: BackupRetentionPolicy = { monthlyMonths: 1, }; +/** + * Instance-wide execution policy. + * + * - `"any"` (default / absent): unrestricted — any environment driver (local, + * ssh, sandbox) may run agents. Preserves single-tenant / local-trusted + * behavior. + * - `"kubernetes"`: force ALL agent execution onto the Kubernetes + * sandbox-provider environment and REFUSE local/in-process execution. Used by + * shared cloud (cloud_tenant) instances so untrusted tenant agents can never + * run in the server process or on an unsandboxed local/ssh adapter. + */ +export type InstanceExecutionMode = "kubernetes" | "any"; + export interface InstanceGeneralSettings { censorUsernameInLogs: boolean; keyboardShortcuts: boolean; feedbackDataSharingPreference: FeedbackDataSharingPreference; backupRetention: BackupRetentionPolicy; + /** + * Execution policy. Absent/`"any"` = unrestricted; `"kubernetes"` forces the + * Kubernetes sandbox provider and denies local/ssh execution. + */ + executionMode?: InstanceExecutionMode; } export interface InstanceExperimentalSettings { diff --git a/packages/shared/src/validators/adapter-registry.test.ts b/packages/shared/src/validators/adapter-registry.test.ts new file mode 100644 index 00000000..567c113e --- /dev/null +++ b/packages/shared/src/validators/adapter-registry.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { adapterRegistrySchema } from "./adapter-registry.js"; + +describe("adapterRegistrySchema", () => { + it("parses a full entry", () => { + const parsed = adapterRegistrySchema.parse([ + { + adapterType: "opencode_local", + runtimeImage: "ghcr.io/paperclipai/agent-runtime-opencode:v1", + envKeys: ["ANTHROPIC_API_KEY"], + allowFqdns: [], + probeCommand: ["opencode", "--version"], + defaultEnv: { ANTHROPIC_BASE_URL: "http://bifrost.bifrost.svc.cluster.local:8080" }, + }, + ]); + expect(parsed[0].adapterType).toBe("opencode_local"); + expect(parsed[0].enabled).toBe(true); // defaulted + expect(parsed[0].defaultEnv?.ANTHROPIC_BASE_URL).toContain("bifrost"); + }); + + it("defaults enabled to true and optional collections to undefined", () => { + const parsed = adapterRegistrySchema.parse([{ adapterType: "pi_local" }]); + expect(parsed[0]).toMatchObject({ adapterType: "pi_local", enabled: true }); + expect(parsed[0].runtimeImage).toBeUndefined(); + }); + + it("rejects an entry with no adapterType", () => { + expect(() => adapterRegistrySchema.parse([{ enabled: true }])).toThrow(); + }); + + it("rejects a non-array", () => { + expect(() => adapterRegistrySchema.parse({ adapterType: "x" })).toThrow(); + }); +}); diff --git a/packages/shared/src/validators/adapter-registry.ts b/packages/shared/src/validators/adapter-registry.ts new file mode 100644 index 00000000..dcab1a93 --- /dev/null +++ b/packages/shared/src/validators/adapter-registry.ts @@ -0,0 +1,17 @@ +import { z } from "zod"; + +export const adapterRegistryEntrySchema = z + .object({ + adapterType: z.string().min(1), + enabled: z.boolean().default(true), + runtimeImage: z.string().optional(), + envKeys: z.array(z.string()).optional(), + allowFqdns: z.array(z.string()).optional(), + probeCommand: z.array(z.string()).optional(), + defaultEnv: z.record(z.string()).optional(), + }) + .strict(); + +export const adapterRegistrySchema = z.array(adapterRegistryEntrySchema); + +export type AdapterRegistryEntryParsed = z.infer; diff --git a/packages/shared/src/validators/instance.ts b/packages/shared/src/validators/instance.ts index e5c235cb..a36e1519 100644 --- a/packages/shared/src/validators/instance.ts +++ b/packages/shared/src/validators/instance.ts @@ -31,6 +31,9 @@ export const instanceGeneralSettingsSchema = z.object({ DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE, ), backupRetention: backupRetentionPolicySchema.default(DEFAULT_BACKUP_RETENTION), + // Execution policy. Absent/"any" = unrestricted; "kubernetes" forces the + // Kubernetes sandbox provider and denies local/ssh execution (cloud_tenant). + executionMode: z.enum(["kubernetes", "any"]).optional(), }).strict(); export const patchInstanceGeneralSettingsSchema = instanceGeneralSettingsSchema.partial(); diff --git a/screenshots/PR-7938-agent-config-forced-kubernetes-missing-env.png b/screenshots/PR-7938-agent-config-forced-kubernetes-missing-env.png new file mode 100644 index 00000000..edef5b53 Binary files /dev/null and b/screenshots/PR-7938-agent-config-forced-kubernetes-missing-env.png differ diff --git a/screenshots/PR-7938-agent-config-forced-kubernetes.png b/screenshots/PR-7938-agent-config-forced-kubernetes.png new file mode 100644 index 00000000..6eef5b52 Binary files /dev/null and b/screenshots/PR-7938-agent-config-forced-kubernetes.png differ diff --git a/server/src/__tests__/adapter-models.test.ts b/server/src/__tests__/adapter-models.test.ts index dc1eba55..b72a7290 100644 --- a/server/src/__tests__/adapter-models.test.ts +++ b/server/src/__tests__/adapter-models.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { models as claudeFallbackModels } from "@paperclipai/adapter-claude-local"; import { resetClaudeModelsCacheForTests } from "@paperclipai/adapter-claude-local/server"; import { models as codexFallbackModels } from "@paperclipai/adapter-codex-local"; @@ -220,4 +220,64 @@ describe("adapter model listing", () => { expect(first.some((model) => model.id === "composer-1")).toBe(true); }); + describe("PAPERCLIP_ADAPTER_MODELS declared models", () => { + afterEach(() => { + delete process.env.PAPERCLIP_ADAPTER_MODELS; + }); + + it("prefers declared env models over adapter discovery", async () => { + process.env.PAPERCLIP_ADAPTER_MODELS = JSON.stringify({ + opencode_local: [ + { id: "tensorix/deepseek/deepseek-chat-v3.1", label: "DeepSeek v3.1" }, + { id: "tensorix/z-ai/glm-4.7" }, + ], + }); + + const models = await listAdapterModels("opencode_local"); + + expect(models).toEqual([ + { id: "tensorix/deepseek/deepseek-chat-v3.1", label: "DeepSeek v3.1" }, + { id: "tensorix/z-ai/glm-4.7", label: "tensorix/z-ai/glm-4.7" }, + ]); + }); + + it("observes env changes between calls (memo keyed by raw env value)", async () => { + process.env.PAPERCLIP_ADAPTER_MODELS = JSON.stringify({ + opencode_local: [{ id: "model-a" }], + }); + expect(await listAdapterModels("opencode_local")).toEqual([ + { id: "model-a", label: "model-a" }, + ]); + + process.env.PAPERCLIP_ADAPTER_MODELS = JSON.stringify({ + opencode_local: [{ id: "model-b" }], + }); + expect(await listAdapterModels("opencode_local")).toEqual([ + { id: "model-b", label: "model-b" }, + ]); + }); + + it("fails soft on malformed values: falls back to adapter models instead of throwing", async () => { + process.env.PAPERCLIP_ADAPTER_MODELS = "{not json"; + process.env.PAPERCLIP_OPENCODE_COMMAND = "__paperclip_missing_opencode_command__"; + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + const models = await listAdapterModels("opencode_local"); + expect(models).toEqual(opencodeFallbackModels); + + // Parsing is memoized per raw value: a second call must not re-log. + const callsAfterFirst = errorSpy.mock.calls.length; + expect(callsAfterFirst).toBeGreaterThan(0); + await listAdapterModels("opencode_local"); + expect(errorSpy.mock.calls.length).toBe(callsAfterFirst); + }); + + it("ignores declared models for adapters not in the map", async () => { + process.env.PAPERCLIP_ADAPTER_MODELS = JSON.stringify({ + opencode_local: [{ id: "model-a" }], + }); + const models = await listAdapterModels("codex_local"); + expect(models).toEqual(codexFallbackModels); + }); + }); }); diff --git a/server/src/__tests__/environment-service.test.ts b/server/src/__tests__/environment-service.test.ts index 30d66c4f..3232301d 100644 --- a/server/src/__tests__/environment-service.test.ts +++ b/server/src/__tests__/environment-service.test.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; -import { eq } from "drizzle-orm"; +import { and, eq } from "drizzle-orm"; import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, @@ -222,6 +222,134 @@ describeEmbeddedPostgres("environmentService leases", () => { expect(rows[0]?.status).toBe("active"); }); + it("ensures, refreshes, and finds a managed Kubernetes sandbox environment", async () => { + const companyId = randomUUID(); + await db.insert(companies).values({ + id: companyId, + name: "Acme", + status: "active", + createdAt: new Date(), + updatedAt: new Date(), + }); + + // No managed k8s env yet. + expect(await svc.findKubernetesEnvironment(companyId)).toBeNull(); + + const created = await svc.ensureKubernetesEnvironment(companyId, { + backend: "job", + inCluster: true, + runtimeClassName: "gvisor", + egressMode: "cilium", + egressAllowFqdns: ["api.anthropic.com"], + }); + + expect(created.driver).toBe("sandbox"); + expect(created.config.provider).toBe("kubernetes"); + expect(created.config.backend).toBe("job"); + expect(created.config.runtimeClassName).toBe("gvisor"); + expect(created.metadata?.managedKubernetesSandbox).toBe(true); + + // Idempotent: second call refreshes config in place, no new row. + const refreshed = await svc.ensureKubernetesEnvironment(companyId, { + backend: "job", + inCluster: true, + egressMode: "cilium", + egressAllowFqdns: ["api.anthropic.com", "api.openai.com"], + }); + expect(refreshed.id).toBe(created.id); + expect(refreshed.config.egressAllowFqdns).toEqual([ + "api.anthropic.com", + "api.openai.com", + ]); + + const found = await svc.findKubernetesEnvironment(companyId); + expect(found?.id).toBe(created.id); + + const rows = await db + .select() + .from(environments) + .where(eq(environments.companyId, companyId)); + expect(rows.filter((row) => row.driver === "sandbox")).toHaveLength(1); + }); + + it("deduplicates concurrent managed Kubernetes environment creation", async () => { + const companyId = randomUUID(); + await db.insert(companies).values({ + id: companyId, + name: "Acme", + status: "active", + createdAt: new Date(), + updatedAt: new Date(), + }); + + // No partial unique index covers sandbox drivers yet, so dedup is + // post-insert convergence (prefer the oldest row, delete the loser). + const results = await Promise.all( + Array.from({ length: 8 }, () => + svc.ensureKubernetesEnvironment(companyId, { inCluster: true, backend: "job" }), + ), + ); + + expect(new Set(results.map((environment) => environment.id)).size).toBe(1); + + const rows = await db + .select() + .from(environments) + .where(and(eq(environments.companyId, companyId), eq(environments.driver, "sandbox"))); + expect(rows).toHaveLength(1); + expect((rows[0]?.metadata as Record)?.managedKubernetesSandbox).toBe(true); + }); + + it("does not treat a non-kubernetes sandbox environment as the managed k8s env", async () => { + const companyId = randomUUID(); + await db.insert(companies).values({ + id: companyId, + name: "Acme", + status: "active", + createdAt: new Date(), + updatedAt: new Date(), + }); + await svc.create(companyId, { + name: "Fake Sandbox", + driver: "sandbox", + config: { provider: "fake", image: "busybox", reuseLease: false }, + }); + + expect(await svc.findKubernetesEnvironment(companyId)).toBeNull(); + }); + + it("ignores a config.provider=kubernetes sandbox env without the managed marker", async () => { + const companyId = randomUUID(); + await db.insert(companies).values({ + id: companyId, + name: "Acme", + status: "active", + createdAt: new Date(), + updatedAt: new Date(), + }); + + // A tenant-created sandbox env with config.provider "kubernetes" but WITHOUT + // the managed metadata marker must NOT be treated as the managed k8s env, + // otherwise it would bypass the operator gVisor runtimeClass / Cilium egress. + await svc.create(companyId, { + name: "Tenant K8s Sandbox", + driver: "sandbox", + config: { provider: "kubernetes", reuseLease: false }, + }); + + expect(await svc.findKubernetesEnvironment(companyId)).toBeNull(); + + // The managed env (created via ensureKubernetesEnvironment) carries the + // marker and is the only one found. + const managed = await svc.ensureKubernetesEnvironment(companyId, { + backend: "job", + inCluster: true, + runtimeClassName: "gvisor", + }); + const found = await svc.findKubernetesEnvironment(companyId); + expect(found?.id).toBe(managed.id); + }); + it("allows multiple SSH environments for the same company", async () => { const companyId = randomUUID(); await db.insert(companies).values({ diff --git a/server/src/__tests__/heartbeat-plugin-environment.test.ts b/server/src/__tests__/heartbeat-plugin-environment.test.ts index 381f4556..5419dc22 100644 --- a/server/src/__tests__/heartbeat-plugin-environment.test.ts +++ b/server/src/__tests__/heartbeat-plugin-environment.test.ts @@ -210,6 +210,11 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => { config: { template: "base" }, runId: run!.id, workspaceMode: "shared_workspace", + // Pins the HEARTBEAT-path lease call forwarding the AGENT's adapter type + // (per-run adapter / mixed-harness envs). environment-runtime.ts has two + // drivers calling environmentAcquireLease; regressions here previously + // shipped by editing only the non-heartbeat one. + adapterType: "codex_local", }); await vi.waitFor(() => { expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", { @@ -427,6 +432,7 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => { config: { template: "new" }, runId: run!.id, workspaceMode: "shared_workspace", + adapterType: "codex_local", }); }, 15_000); }); diff --git a/server/src/__tests__/plugin-database.test.ts b/server/src/__tests__/plugin-database.test.ts index 82b455a9..98753b41 100644 --- a/server/src/__tests__/plugin-database.test.ts +++ b/server/src/__tests__/plugin-database.test.ts @@ -168,6 +168,26 @@ describe("buildPluginWorkerEnv", () => { }); }); + it("passes in-cluster Kubernetes service-discovery vars to environment driver plugins", () => { + const env = buildPluginWorkerEnv({ + manifest: { capabilities: ["environment.drivers.register"] }, + instanceInfo, + processEnv: { + KUBERNETES_SERVICE_HOST: "10.0.0.1", + KUBERNETES_SERVICE_PORT: "443", + KUBERNETES_SERVICE_PORT_HTTPS: " ", + AWS_SECRET_ACCESS_KEY: "aws-secret", + }, + }); + + expect(env).toEqual({ + PAPERCLIP_DEPLOYMENT_MODE: "authenticated", + PAPERCLIP_DEPLOYMENT_EXPOSURE: "public", + KUBERNETES_SERVICE_HOST: "10.0.0.1", + KUBERNETES_SERVICE_PORT: "443", + }); + }); + it("does not pass provider keys to non-environment plugins", () => { const env = buildPluginWorkerEnv({ manifest: { capabilities: ["ui.slots.register"] }, diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 3cdab7c4..f51c09f3 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -143,6 +143,7 @@ vi.mock("../services/index.js", () => ({ humanGrantsInserted: 0, })), feedbackService: feedbackServiceFactoryMock, + bootstrapExecutionPolicyFromEnv: vi.fn(async () => null), heartbeatService: vi.fn(() => ({ reapOrphanedRuns: vi.fn(async () => undefined), promoteDueScheduledRetries: vi.fn(async () => ({ promoted: 0, runIds: [] })), diff --git a/server/src/adapters/registry.ts b/server/src/adapters/registry.ts index 5bb261b8..55bce77b 100644 --- a/server/src/adapters/registry.ts +++ b/server/src/adapters/registry.ts @@ -4,6 +4,7 @@ import type { AdapterRuntimeCommandSpec, ServerAdapterModule, } from "./types.js"; +import { parseAdapterModelsEnv } from "../services/adapter-models-env.js"; import { buildSandboxNpmInstallCommand, getAdapterSessionManagement, @@ -654,7 +655,42 @@ export function getServerAdapter(type: string): ServerAdapterModule { return findActiveServerAdapter(type) ?? processAdapter; } +/** + * Memoized view of PAPERCLIP_ADAPTER_MODELS, keyed by the raw env string so + * tests (and live env mutation) that change the variable are still observed. + * Parsing happens at most once per distinct raw value instead of per + * `listAdapterModels` request, and malformed values fail SOFT here: we log the + * parse error once (per distinct raw value) and fall back to adapter-discovered + * models rather than throwing at request time. + */ +let adapterModelsEnvCache: { + raw: string | undefined; + value: ReturnType; +} | null = null; + +function getDeclaredAdapterModels(): ReturnType { + const raw = process.env.PAPERCLIP_ADAPTER_MODELS; + if (adapterModelsEnvCache && adapterModelsEnvCache.raw === raw) { + return adapterModelsEnvCache.value; + } + let value: ReturnType = null; + try { + value = parseAdapterModelsEnv(process.env); + } catch (err) { + console.error( + "[paperclip] Invalid PAPERCLIP_ADAPTER_MODELS; ignoring declared model lists:", + err, + ); + } + adapterModelsEnvCache = { raw, value }; + return value; +} + export async function listAdapterModels(type: string): Promise<{ id: string; label: string }[]> { + const declaredModels = getDeclaredAdapterModels(); + if (declaredModels && declaredModels[type]?.length) { + return declaredModels[type].map((m) => ({ id: m.id, label: m.label ?? m.id })); + } const adapter = findActiveServerAdapter(type); if (!adapter) return []; if (adapter.listModels) { diff --git a/server/src/app.ts b/server/src/app.ts index e91bc00a..b02b963d 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -471,7 +471,67 @@ export async function createApp( lifecycle, async (pluginId) => (await pluginRegistry.getById(pluginId))?.packagePath ?? null, ); - void loader.loadAll().then((result) => { + // Auto-install the bundled kubernetes sandbox-provider plugin so the + // "kubernetes" sandbox provider is registered for agent runs. The plugin is + // excluded from the pnpm workspace and built standalone into the image (see + // Dockerfile), then installed here from its local path. This runs BEFORE + // loadAll() so loadAll() can activate it in the same startup pass. + // + // SAFETY (invariant B): this is fully fail-safe. Any failure (missing path, + // install error, load error) is caught, logged, and swallowed so the server + // ALWAYS finishes booting. A degraded boot (no kubernetes provider, agents + // cannot run) is strictly preferable to a crash loop. + const ensureBundledKubernetesPlugin = async (): Promise => { + const KUBERNETES_PLUGIN_KEY = "paperclip.kubernetes-sandbox-provider"; + const pluginPath = + process.env["PAPERCLIP_KUBERNETES_PLUGIN_PATH"] ?? + "/app/packages/plugins/sandbox-providers/kubernetes"; + try { + // Idempotent: skip if already installed (any non-uninstalled status). + const existing = await pluginRegistry.getByKey(KUBERNETES_PLUGIN_KEY); + if (existing) { + logger.info( + { pluginKey: KUBERNETES_PLUGIN_KEY, status: existing.status }, + "kubernetes sandbox plugin already installed; skipping auto-install", + ); + return; + } + // Skip silently when the bundle is absent (e.g. local dev or an image + // built without the plugin). Not an error condition. + if (!fs.existsSync(path.join(pluginPath, "dist", "manifest.js"))) { + logger.info( + { pluginPath }, + "kubernetes sandbox plugin bundle not present; skipping auto-install", + ); + return; + } + logger.info({ pluginPath }, "auto-installing bundled kubernetes sandbox plugin"); + const discovered = await loader.installPlugin({ localPath: pluginPath }); + if (!discovered.manifest) { + logger.error("kubernetes sandbox plugin installed but manifest is missing"); + return; + } + // Transition installed -> ready and activate the worker. + const installed = await pluginRegistry.getByKey(discovered.manifest.id); + if (installed) { + await lifecycle.load(installed.id); + logger.info( + { pluginId: installed.id, pluginKey: installed.pluginKey }, + "kubernetes sandbox plugin auto-installed and loaded", + ); + } else { + logger.error("kubernetes sandbox plugin installed but not found in registry"); + } + } catch (err) { + logger.error( + { err }, + "Failed to auto-install the kubernetes sandbox plugin; continuing boot (degraded: kubernetes provider unavailable)", + ); + } + }; + void ensureBundledKubernetesPlugin() + .then(() => loader.loadAll()) + .then((result) => { if (!result) return; for (const loaded of result.results) { if (devWatcher && loaded.success && loaded.plugin.packagePath) { diff --git a/server/src/index.ts b/server/src/index.ts index a81e50b5..6708c535 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -32,12 +32,17 @@ import { setupLiveEventsWebSocketServer } from "./realtime/live-events-ws.js"; import { feedbackService, backfillPrincipalAccessCompatibility, + bootstrapExecutionPolicyFromEnv, heartbeatService, instanceSettingsService, reconcileCloudUpstreamRunsOnStartup, reconcilePersistedRuntimeServicesOnStartup, routineService, } from "./services/index.js"; +import { + parseAdapterRegistryEnv, + reconcileAdapterAvailability, +} from "./services/adapter-registry-bootstrap.js"; import { createFeedbackTraceShareClientFromConfig } from "./services/feedback-share-client.js"; import { buildRuntimeApiCandidateUrls, choosePrimaryRuntimeApiUrl } from "./runtime-api.js"; import { createPluginWorkerManager } from "./services/plugin-worker-manager.js"; @@ -716,6 +721,27 @@ export async function startServer(): Promise { logger.error({ err }, "startup reconciliation of cloud upstream runs failed"); }); + // Force the instance onto the Kubernetes sandbox provider when configured via + // env (PAPERCLIP_EXECUTION_MODE=kubernetes). Runs BEFORE the heartbeat resumes + // queued runs so the policy + managed k8s environments are in place. A bad + // PAPERCLIP_EXECUTION_MODE / PAPERCLIP_K8S_* value throws and fails startup + // (fail-loud) rather than silently allowing local execution. + try { + const policyResult = await bootstrapExecutionPolicyFromEnv(db as any); + if (policyResult) { + logger.warn( + { + executionMode: policyResult.executionMode, + companiesConfigured: policyResult.companiesConfigured, + }, + "forced execution policy applied at startup", + ); + } + } catch (err) { + logger.error({ err }, "failed to apply forced execution policy from environment"); + throw err; + } + if (config.heartbeatSchedulerEnabled) { const heartbeat = heartbeatService(db as any, { pluginWorkerManager }); const routines = routineService(db as any, { pluginWorkerManager }); @@ -868,6 +894,18 @@ export async function startServer(): Promise { const { waitForExternalAdapters } = await import("./adapters/registry.js"); await waitForExternalAdapters(); + // Reconcile the agent-creation picker to the declaratively-configured adapter + // set (PAPERCLIP_ADAPTERS). Must run after external adapters are loaded so the + // known-adapter list is complete. Fail loud on misconfig (a declared adapter + // with no implementation), consistent with the execution-policy bootstrap: + // log the structured error, then rethrow to fail startup. + try { + reconcileAdapterAvailability(parseAdapterRegistryEnv()); + } catch (err) { + logger.error({ err }, "failed to reconcile adapter availability from PAPERCLIP_ADAPTERS"); + throw err; + } + await new Promise((resolveListen, rejectListen) => { const onError = (err: Error) => { server.off("error", onError); diff --git a/server/src/services/adapter-models-env.test.ts b/server/src/services/adapter-models-env.test.ts new file mode 100644 index 00000000..8b32b84a --- /dev/null +++ b/server/src/services/adapter-models-env.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import { parseAdapterModelsEnv } from "./adapter-models-env.js"; + +const ENV = JSON.stringify({ + opencode_local: [ + { id: "tensorix/deepseek/deepseek-chat-v3.1", label: "DeepSeek v3.1" }, + { id: "tensorix/z-ai/glm-4.7", label: "GLM 4.7" }, + ], +}); + +describe("parseAdapterModelsEnv", () => { + it("returns null when unset", () => { + expect(parseAdapterModelsEnv({})).toBeNull(); + }); + it("parses the per-adapter model map", () => { + const m = parseAdapterModelsEnv({ PAPERCLIP_ADAPTER_MODELS: ENV }); + expect(m?.opencode_local?.[0]).toEqual({ id: "tensorix/deepseek/deepseek-chat-v3.1", label: "DeepSeek v3.1" }); + expect(m?.opencode_local?.length).toBe(2); + }); + it("defaults label to id when omitted", () => { + const m = parseAdapterModelsEnv({ PAPERCLIP_ADAPTER_MODELS: JSON.stringify({ pi_local: [{ id: "tensorix/x/y" }] }) }); + expect(m?.pi_local?.[0]).toEqual({ id: "tensorix/x/y", label: "tensorix/x/y" }); + }); + it("throws on invalid JSON (fail loud)", () => { + expect(() => parseAdapterModelsEnv({ PAPERCLIP_ADAPTER_MODELS: "{bad" })).toThrow(/PAPERCLIP_ADAPTER_MODELS/); + }); + it("throws when an entry lacks a string id", () => { + expect(() => parseAdapterModelsEnv({ PAPERCLIP_ADAPTER_MODELS: JSON.stringify({ a: [{ label: "x" }] }) })).toThrow(); + }); +}); diff --git a/server/src/services/adapter-models-env.ts b/server/src/services/adapter-models-env.ts new file mode 100644 index 00000000..43ddb63b --- /dev/null +++ b/server/src/services/adapter-models-env.ts @@ -0,0 +1,40 @@ +export interface AdapterModelEntry { + id: string; + label?: string; +} + +/** + * Per-adapter model list supplied by the operator via env, so the agent model + * picker can offer models the server cannot CLI-discover (e.g. gateway models). + * JSON object: adapterType -> [{ id, label? }]. Returns null when unset; throws + * loudly on malformed input. + */ +export function parseAdapterModelsEnv( + env: Record = process.env, +): Record | null { + const raw = env.PAPERCLIP_ADAPTER_MODELS?.trim(); + if (!raw) return null; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (e) { + throw new Error(`PAPERCLIP_ADAPTER_MODELS must be valid JSON: ${(e as Error).message}`); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("PAPERCLIP_ADAPTER_MODELS must be a JSON object mapping adapterType to an array of {id,label}"); + } + const out: Record = {}; + for (const [type, list] of Object.entries(parsed as Record)) { + if (!Array.isArray(list)) { + throw new Error(`PAPERCLIP_ADAPTER_MODELS[${type}] must be an array`); + } + out[type] = list.map((m) => { + const o = m as Record; + if (typeof o.id !== "string" || !o.id) { + throw new Error(`PAPERCLIP_ADAPTER_MODELS[${type}] entries require a non-empty string id`); + } + return { id: o.id, label: typeof o.label === "string" ? o.label : o.id }; + }); + } + return out; +} diff --git a/server/src/services/adapter-registry-bootstrap.reconcile.test.ts b/server/src/services/adapter-registry-bootstrap.reconcile.test.ts new file mode 100644 index 00000000..fb406926 --- /dev/null +++ b/server/src/services/adapter-registry-bootstrap.reconcile.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it, vi } from "vitest"; + +const setAdapterDisabled = vi.fn(); + +vi.mock("../adapters/registry.js", async (orig) => ({ + ...(await orig()), + listServerAdapters: () => [ + { type: "claude_local" }, + { type: "opencode_local" }, + { type: "pi_local" }, + ], +})); + +vi.mock("./adapter-plugin-store.js", () => ({ + setAdapterDisabled: (type: string, disabled: boolean) => setAdapterDisabled(type, disabled), +})); + +const { reconcileAdapterAvailability } = await import("./adapter-registry-bootstrap.js"); + +describe("reconcileAdapterAvailability", () => { + it("is a no-op when registry is null", () => { + setAdapterDisabled.mockReset(); + expect(reconcileAdapterAvailability(null)).toEqual({ enabled: [], disabled: [] }); + expect(setAdapterDisabled).not.toHaveBeenCalled(); + }); + + it("enables declared, disables everything else (e.g. drops claude_local)", () => { + setAdapterDisabled.mockReset(); + const result = reconcileAdapterAvailability([ + { adapterType: "opencode_local", enabled: true }, + ]); + expect(result.enabled).toEqual(["opencode_local"]); + expect(result.disabled.sort()).toEqual(["claude_local", "pi_local"]); + expect(setAdapterDisabled).toHaveBeenCalledWith("claude_local", true); + expect(setAdapterDisabled).toHaveBeenCalledWith("opencode_local", false); + }); + + it("throws when a declared adapter has no installed implementation", () => { + setAdapterDisabled.mockReset(); + expect(() => + reconcileAdapterAvailability([{ adapterType: "ghost_adapter", enabled: true }]), + ).toThrow(/no installed adapter: ghost_adapter/); + }); +}); diff --git a/server/src/services/adapter-registry-bootstrap.test.ts b/server/src/services/adapter-registry-bootstrap.test.ts new file mode 100644 index 00000000..c75a121d --- /dev/null +++ b/server/src/services/adapter-registry-bootstrap.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { parseAdapterRegistryEnv } from "./adapter-registry-bootstrap.js"; + +const ENTRY = JSON.stringify([ + { adapterType: "opencode_local", runtimeImage: "img", envKeys: ["ANTHROPIC_API_KEY"], allowFqdns: [], probeCommand: ["opencode", "--version"], defaultEnv: { ANTHROPIC_BASE_URL: "http://bifrost:8080" } }, +]); + +describe("parseAdapterRegistryEnv", () => { + it("returns null when neither env nor file is set", () => { + expect(parseAdapterRegistryEnv({})).toBeNull(); + }); + + it("parses inline PAPERCLIP_ADAPTERS JSON", () => { + const r = parseAdapterRegistryEnv({ PAPERCLIP_ADAPTERS: ENTRY }); + expect(r).toHaveLength(1); + expect(r?.[0].adapterType).toBe("opencode_local"); + expect(r?.[0].enabled).toBe(true); + }); + + it("throws on malformed JSON (fail loud)", () => { + expect(() => parseAdapterRegistryEnv({ PAPERCLIP_ADAPTERS: "{not json" })).toThrow( + /PAPERCLIP_ADAPTERS/, + ); + }); + + it("throws on schema-invalid content (fail loud)", () => { + expect(() => + parseAdapterRegistryEnv({ PAPERCLIP_ADAPTERS: JSON.stringify([{ enabled: true }]) }), + ).toThrow(/PAPERCLIP_ADAPTERS/); + }); +}); diff --git a/server/src/services/adapter-registry-bootstrap.ts b/server/src/services/adapter-registry-bootstrap.ts new file mode 100644 index 00000000..c8e7f4b0 --- /dev/null +++ b/server/src/services/adapter-registry-bootstrap.ts @@ -0,0 +1,97 @@ +/** + * Declarative adapter-registry bootstrap. + * + * One source (`PAPERCLIP_ADAPTERS` inline JSON, or `PAPERCLIP_ADAPTERS_FILE` a + * path to a JSON file) feeds two consumers: + * 1. Availability: reconcile the file-backed disabled-set so the picker shows + * exactly the declared, enabled set (runs for any instance). + * 2. k8s runtime: the same registry rides on the Kubernetes environment config + * (see execution-policy-bootstrap) so the plugin resolves runtime + * image/envKeys/allowFqdns/probe/defaultEnv from it. + * + * Parsing is pure + fails loud on malformed/invalid config, mirroring + * execution-policy-bootstrap. + */ +import fs from "node:fs"; +import { adapterRegistrySchema, type AdapterRegistryEntryParsed } from "@paperclipai/shared"; +import { logger } from "../middleware/logger.js"; +import { listServerAdapters } from "../adapters/registry.js"; +import { setAdapterDisabled } from "./adapter-plugin-store.js"; + +export type AdapterRegistryEnv = Record; + +/** + * Parse the declarative registry from env. Returns null when unconfigured + * (built-in defaults; local/OSS unchanged). Throws on malformed/invalid input. + */ +export function parseAdapterRegistryEnv( + env: AdapterRegistryEnv = process.env, +): AdapterRegistryEntryParsed[] | null { + const inline = env.PAPERCLIP_ADAPTERS?.trim(); + const filePath = env.PAPERCLIP_ADAPTERS_FILE?.trim(); + if (!inline && !filePath) return null; + + let rawText: string; + if (inline) { + rawText = inline; + } else { + try { + rawText = fs.readFileSync(filePath as string, "utf-8"); + } catch (err) { + throw new Error( + `PAPERCLIP_ADAPTERS_FILE could not be read at "${filePath}": ${(err as Error).message}`, + ); + } + } + + let parsed: unknown; + try { + parsed = JSON.parse(rawText); + } catch (err) { + throw new Error(`PAPERCLIP_ADAPTERS must be valid JSON: ${(err as Error).message}`); + } + + const result = adapterRegistrySchema.safeParse(parsed); + if (!result.success) { + throw new Error( + `PAPERCLIP_ADAPTERS failed validation: ${result.error.issues + .map((i) => `${i.path.join(".")}: ${i.message}`) + .join("; ")}`, + ); + } + return result.data; +} + +/** + * Reconcile availability: every known server adapter NOT enabled in the declared + * registry is disabled; declared+enabled ones are enabled. Throws if a declared + * adapterType has no installed adapter (cannot offer a harness with no + * implementation). No-op when `registry` is null. + */ +export function reconcileAdapterAvailability( + registry: AdapterRegistryEntryParsed[] | null, +): { enabled: string[]; disabled: string[] } { + if (!registry) return { enabled: [], disabled: [] }; + + const knownTypes = new Set(listServerAdapters().map((a) => a.type)); + const declared = new Map(registry.map((e) => [e.adapterType, e])); + + const missing = [...declared.keys()].filter((t) => !knownTypes.has(t)); + if (missing.length > 0) { + throw new Error( + `PAPERCLIP_ADAPTERS declares adapter type(s) with no installed adapter: ${missing.join(", ")}`, + ); + } + + const enabled: string[] = []; + const disabled: string[] = []; + for (const type of knownTypes) { + const entry = declared.get(type); + const shouldEnable = entry !== undefined && entry.enabled !== false; + setAdapterDisabled(type, !shouldEnable); + (shouldEnable ? enabled : disabled).push(type); + } + + logger.info({ enabled, disabled }, "reconciled adapter availability from PAPERCLIP_ADAPTERS"); + return { enabled, disabled }; +} diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index 9b0a2066..693b57c6 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -208,6 +208,7 @@ export function environmentRunOrchestrator( issueId: string | null; heartbeatRunId: string; persistedExecutionWorkspace: Pick | null; + adapterType: string | null; }): Promise { try { return await environmentRuntime.acquireRunLease(input); @@ -282,6 +283,7 @@ export function environmentRunOrchestrator( issueId: input.issueId, heartbeatRunId: input.heartbeatRunId, persistedExecutionWorkspace: input.persistedExecutionWorkspace, + adapterType: input.adapterType ?? null, }); // Step 3: Log lease acquisition activity diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index d876b6d3..08451082 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -112,6 +112,13 @@ export interface EnvironmentDriverAcquireInput { heartbeatRunId: string | null; executionWorkspaceId: string | null; executionWorkspaceMode: ExecutionWorkspace["mode"] | null; + /** + * The harness/adapter type for this run (the agent's adapter). Drivers that + * materialize a per-run sandbox use it to select the runtime image so a single + * environment can serve mixed harnesses; null falls back to the environment's + * configured default adapter. + */ + adapterType: string | null; } export interface EnvironmentDriverReleaseInput { @@ -490,6 +497,14 @@ function createSandboxEnvironmentDriver( // a well-formed identifier. runId: input.heartbeatRunId ?? randomUUID(), workspaceMode: input.executionWorkspaceMode ?? undefined, + // The agent's harness for THIS run, so the plugin picks the matching + // runtime image (per-run adapter, mixed-harness environments). + // NOTE: environment-runtime.ts has TWO drivers calling + // environmentAcquireLease; this plugin-sandbox one is the HEARTBEAT + // path. Omitting adapterType here silently falls back to the + // environment's default adapter image (a pi agent then runs in the + // opencode image and the harness binary is missing at exec time). + adapterType: input.adapterType ?? undefined, }, resolvePluginSandboxRpcTimeoutMs(workerConfig), ); @@ -898,6 +913,7 @@ function createPluginEnvironmentDriver( config: parsed.config.driverConfig, runId: input.heartbeatRunId ?? randomUUID(), workspaceMode: input.executionWorkspaceMode ?? undefined, + adapterType: input.adapterType ?? undefined, }); return await environmentsSvc.acquireLease({ @@ -1113,6 +1129,8 @@ export function environmentRuntimeService( /** Null for ad-hoc invocations (e.g. operator-initiated `Test` probes). */ heartbeatRunId: string | null; persistedExecutionWorkspace: Pick | null; + /** The agent's adapter type for this run (mixed-harness environments). */ + adapterType?: string | null; }): Promise { if (input.environment.status !== "active") { throw new Error(`Environment "${input.environment.name}" is not active.`); @@ -1129,6 +1147,7 @@ export function environmentRuntimeService( heartbeatRunId: input.heartbeatRunId, executionWorkspaceId: leaseContext.executionWorkspaceId, executionWorkspaceMode: leaseContext.executionWorkspaceMode, + adapterType: input.adapterType ?? null, }); return { diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts index a94e5aa4..e4287a3c 100644 --- a/server/src/services/environments.ts +++ b/server/src/services/environments.ts @@ -1,4 +1,4 @@ -import { and, desc, eq, sql } from "drizzle-orm"; +import { and, asc, desc, eq, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { environmentLeases, environments } from "@paperclipai/db"; import { @@ -22,6 +22,43 @@ const DEFAULT_LOCAL_ENVIRONMENT_NAME = "Local"; const DEFAULT_LOCAL_ENVIRONMENT_DESCRIPTION = "Default execution environment for Paperclip runs on this machine."; +const DEFAULT_KUBERNETES_ENVIRONMENT_NAME = "Kubernetes Sandbox"; +const DEFAULT_KUBERNETES_ENVIRONMENT_DESCRIPTION = + "Managed Kubernetes sandbox environment for hosted tenant execution."; +/** Provider key (== plugin driverKey) of the first-party Kubernetes sandbox provider. */ +const KUBERNETES_PROVIDER_KEY = "kubernetes"; +/** Metadata marker for the company's managed-by-config Kubernetes sandbox environment. */ +const KUBERNETES_MANAGED_MARKER = "managedKubernetesSandbox"; + +/** + * Configuration accepted by `ensureKubernetesEnvironment`. Mirrors the keys of + * the kubernetes sandbox-provider `configSchema` that an operator typically + * pins for a hosted cloud instance. Stored verbatim in `environment.config` + * (the plugin validates/defaults it via `kubernetesProviderConfigSchema` at + * lease time); `provider` is always forced to "kubernetes". + */ +export interface KubernetesEnvironmentConfigInput { + backend?: "sandbox-cr" | "job"; + inCluster?: boolean; + runtimeClassName?: string; + egressMode?: "cilium" | "standard"; + egressAllowFqdns?: string[]; + egressAllowCidrs?: string[]; + namespacePrefix?: string; + imageRegistry?: string; + adapterType?: string; + /** + * Sandbox lease RPC timeout in milliseconds. Read at lease time by + * `resolvePluginSandboxRpcTimeoutMs` to extend the worker-manager call + * timeout when acquiring a lease may take minutes (e.g. a cold node + * scale-up on an autoscale-to-zero pool). Stored verbatim in the + * environment config and validated by the sandbox config schema. + */ + timeoutMs?: number; + adapters?: import("@paperclipai/shared").AdapterRegistryEntry[]; + [key: string]: unknown; +} + function cloneRecord(value: unknown, fallback: Record | null = null): Record | null { if (!value || typeof value !== "object" || Array.isArray(value)) return fallback; return { ...(value as Record) }; @@ -147,6 +184,130 @@ export function environmentService(db: Db) { return toEnvironment(existing); }, + /** + * Idempotently ensure a managed Kubernetes sandbox environment exists for a + * company, configured from instance/operator-supplied config. Mirrors + * `ensureLocalEnvironment`, but there is no DB unique index for sandbox + * drivers, so idempotency is by metadata marker + driver lookup. + * + * The environment is `driver: "sandbox"` with `config.provider: + * "kubernetes"` so it resolves to the first-party Kubernetes sandbox + * provider. On subsequent calls the config is refreshed (so operators can + * update egress/runtimeClass via gitops without recreating the row). + */ + ensureKubernetesEnvironment: async ( + companyId: string, + config: KubernetesEnvironmentConfigInput, + ): Promise => { + const desiredConfig: Record = { + ...config, + provider: KUBERNETES_PROVIDER_KEY, + }; + const desiredMetadata: Record = { + managedByPaperclip: true, + [KUBERNETES_MANAGED_MARKER]: true, + }; + + const existing = await db + .select() + .from(environments) + .where(and(eq(environments.companyId, companyId), eq(environments.driver, "sandbox"))) + .then((rows) => + rows.find( + (row) => + (row.metadata as Record | null)?.[KUBERNETES_MANAGED_MARKER] === true, + ) ?? null, + ); + + const now = new Date(); + if (existing) { + const updated = await db + .update(environments) + .set({ + config: desiredConfig, + metadata: { ...(existing.metadata ?? {}), ...desiredMetadata }, + status: "active", + updatedAt: now, + }) + .where(eq(environments.id, existing.id)) + .returning() + .then((rows) => rows[0] ?? existing); + return toEnvironment(updated); + } + + const row = await db + .insert(environments) + .values({ + companyId, + name: DEFAULT_KUBERNETES_ENVIRONMENT_NAME, + description: DEFAULT_KUBERNETES_ENVIRONMENT_DESCRIPTION, + driver: "sandbox", + status: "active", + config: desiredConfig, + metadata: desiredMetadata, + createdAt: now, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0] ?? null); + if (!row) { + throw new Error("Failed to ensure kubernetes environment"); + } + + // Concurrency: the schema's (companyId, driver) unique index is partial + // on driver='local' only, so there is no DB constraint stopping two + // simultaneous callers (e.g. concurrent heartbeats lazily provisioning a + // new company) from both inserting a managed k8s row. Until a partial + // unique index on (companyId, driver) WHERE the managed marker exists is + // added via migration (the proper long-term fix), converge here: re-read, + // deterministically prefer the oldest managed row, and delete our own + // insert if it lost the race. Both racers compute the same winner, so + // duplicates self-heal instead of persisting. + const winner = await db + .select() + .from(environments) + .where(and(eq(environments.companyId, companyId), eq(environments.driver, "sandbox"))) + .orderBy(asc(environments.createdAt), asc(environments.id)) + .then( + (rows) => + rows.find( + (candidate) => + (candidate.metadata as Record | null)?.[ + KUBERNETES_MANAGED_MARKER + ] === true, + ) ?? null, + ); + if (winner && winner.id !== row.id) { + await db.delete(environments).where(eq(environments.id, row.id)); + return toEnvironment(winner); + } + return toEnvironment(row); + }, + + /** + * Find an active Kubernetes sandbox environment for a company, if one + * exists. Read-only counterpart to `ensureKubernetesEnvironment` used by the + * per-run execution guard (which must not silently create config-less envs). + */ + findKubernetesEnvironment: async (companyId: string): Promise => { + const rows = await db + .select() + .from(environments) + .where( + and( + eq(environments.companyId, companyId), + eq(environments.driver, "sandbox"), + eq(environments.status, "active"), + ), + ) + .orderBy(desc(environments.updatedAt)); + const match = rows.find( + (row) => + (row.metadata as Record | null)?.[KUBERNETES_MANAGED_MARKER] === true, + ); + return match ? toEnvironment(match) : null; + }, + create: async (companyId: string, input: CreateEnvironment): Promise => { const now = new Date(); const row = await db diff --git a/server/src/services/execution-allowlist.test.ts b/server/src/services/execution-allowlist.test.ts new file mode 100644 index 00000000..fc53c7e5 --- /dev/null +++ b/server/src/services/execution-allowlist.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from "vitest"; +import { + KUBERNETES_PROVIDER_KEY, + evaluateExecutionAllowlist, + type ExecutionEnvironmentCandidate, +} from "./execution-allowlist.js"; + +const localEnv: ExecutionEnvironmentCandidate = { + driver: "local", + provider: null, +}; + +const kubernetesEnv: ExecutionEnvironmentCandidate = { + driver: "sandbox", + provider: KUBERNETES_PROVIDER_KEY, +}; + +const fakeSandboxEnv: ExecutionEnvironmentCandidate = { + driver: "sandbox", + provider: "fake", +}; + +const sshEnv: ExecutionEnvironmentCandidate = { + driver: "ssh", + provider: null, +}; + +describe("evaluateExecutionAllowlist", () => { + describe('executionMode "any" (unrestricted, default)', () => { + it("allows the local environment", () => { + const result = evaluateExecutionAllowlist({ executionMode: "any" }, localEnv); + expect(result.allowed).toBe(true); + }); + + it("allows the kubernetes sandbox environment", () => { + const result = evaluateExecutionAllowlist({ executionMode: "any" }, kubernetesEnv); + expect(result.allowed).toBe(true); + }); + + it("allows a non-kubernetes sandbox environment", () => { + const result = evaluateExecutionAllowlist({ executionMode: "any" }, fakeSandboxEnv); + expect(result.allowed).toBe(true); + }); + + it("treats absent executionMode as unrestricted", () => { + expect(evaluateExecutionAllowlist({}, localEnv).allowed).toBe(true); + expect(evaluateExecutionAllowlist({ executionMode: undefined }, localEnv).allowed).toBe(true); + }); + }); + + describe('executionMode "kubernetes" (forced sandbox)', () => { + it("allows ONLY a kubernetes sandbox_provider environment", () => { + const result = evaluateExecutionAllowlist({ executionMode: "kubernetes" }, kubernetesEnv); + expect(result.allowed).toBe(true); + }); + + it("DENIES the local environment", () => { + const result = evaluateExecutionAllowlist({ executionMode: "kubernetes" }, localEnv); + expect(result.allowed).toBe(false); + if (!result.allowed) { + expect(result.reason).toMatch(/kubernetes/i); + expect(result.deniedDriver).toBe("local"); + } + }); + + it("DENIES an ssh environment", () => { + const result = evaluateExecutionAllowlist({ executionMode: "kubernetes" }, sshEnv); + expect(result.allowed).toBe(false); + }); + + it("DENIES a non-kubernetes sandbox provider (e.g. fake)", () => { + const result = evaluateExecutionAllowlist({ executionMode: "kubernetes" }, fakeSandboxEnv); + expect(result.allowed).toBe(false); + if (!result.allowed) { + expect(result.deniedProvider).toBe("fake"); + } + }); + + it("DENIES a sandbox driver with no provider", () => { + const result = evaluateExecutionAllowlist( + { executionMode: "kubernetes" }, + { driver: "sandbox", provider: null }, + ); + expect(result.allowed).toBe(false); + }); + }); + + describe("isExecutionForcedToKubernetes helper", () => { + it("reflects the policy", async () => { + const { isExecutionForcedToKubernetes } = await import("./execution-allowlist.js"); + expect(isExecutionForcedToKubernetes({ executionMode: "kubernetes" })).toBe(true); + expect(isExecutionForcedToKubernetes({ executionMode: "any" })).toBe(false); + expect(isExecutionForcedToKubernetes({})).toBe(false); + }); + }); +}); diff --git a/server/src/services/execution-allowlist.ts b/server/src/services/execution-allowlist.ts new file mode 100644 index 00000000..c37e6b66 --- /dev/null +++ b/server/src/services/execution-allowlist.ts @@ -0,0 +1,103 @@ +/** + * Pure execution-allowlist guard. + * + * Decides whether a candidate execution environment is permitted to run an + * agent, given the instance-level execution policy. This is security-critical: + * on a shared cloud instance we FORCE all untrusted tenant agents onto the + * Kubernetes sandbox-provider and REFUSE local/in-process execution so that a + * tenant agent can never run inside the server process or on an unsandboxed + * local/ssh adapter. + * + * The merged tree's environment model represents the Kubernetes sandbox as a + * core `driver: "sandbox"` environment whose `config.provider` is the plugin's + * `driverKey` ("kubernetes", `kind: "sandbox_provider"`). The local default is + * `driver: "local"`. This module knows nothing about the DB or heartbeat — it + * just maps (driver, provider, policy) -> allow/deny so it is trivially + * unit-testable. + */ + +/** Provider key (== plugin driverKey) of the first-party Kubernetes sandbox provider. */ +export const KUBERNETES_PROVIDER_KEY = "kubernetes" as const; + +/** + * Instance execution policy as read from instance general settings. + * + * - `"any"` / absent: unrestricted — any environment driver is allowed (the + * default, preserves single-tenant / local-trusted behavior). + * - `"kubernetes"`: force the Kubernetes sandbox provider; deny local, ssh, and + * any non-kubernetes sandbox provider. + */ +export interface ExecutionPolicy { + executionMode?: "kubernetes" | "any"; +} + +/** + * The minimal shape of the selected/candidate environment the guard needs. + * `driver` is the core `EnvironmentDriver`; `provider` is the sandbox provider + * key (== plugin driverKey) for `driver: "sandbox"` environments, else null. + */ +export interface ExecutionEnvironmentCandidate { + driver: string; + provider: string | null | undefined; +} + +export type ExecutionAllowlistDecision = + | { allowed: true } + | { + allowed: false; + reason: string; + deniedDriver: string; + deniedProvider: string | null; + }; + +/** True when the policy forces all execution onto the Kubernetes sandbox. */ +export function isExecutionForcedToKubernetes(policy: ExecutionPolicy | null | undefined): boolean { + return policy?.executionMode === "kubernetes"; +} + +/** + * True iff the candidate environment is the Kubernetes sandbox provider, i.e. a + * core `sandbox` driver whose provider key is "kubernetes". + */ +export function isKubernetesSandboxEnvironment( + candidate: ExecutionEnvironmentCandidate, +): boolean { + return candidate.driver === "sandbox" && candidate.provider === KUBERNETES_PROVIDER_KEY; +} + +/** + * Decide whether the candidate environment may run under the given policy. + * + * When `executionMode === "kubernetes"`, ONLY a `sandbox_provider` driver with + * provider/driverKey "kubernetes" is allowed; a `local` driver (or any non-k8s + * sandbox provider, or ssh, or plugin) is DENIED. Otherwise everything is + * allowed. + */ +export function evaluateExecutionAllowlist( + policy: ExecutionPolicy | null | undefined, + candidate: ExecutionEnvironmentCandidate, +): ExecutionAllowlistDecision { + if (!isExecutionForcedToKubernetes(policy)) { + return { allowed: true }; + } + + if (isKubernetesSandboxEnvironment(candidate)) { + return { allowed: true }; + } + + const provider = candidate.provider ?? null; + const target = + candidate.driver === "sandbox" + ? `sandbox provider "${provider ?? "(none)"}"` + : `"${candidate.driver}" driver`; + + return { + allowed: false, + reason: + `Instance execution policy requires the Kubernetes sandbox provider ` + + `(executionMode=kubernetes), but the resolved environment uses the ${target}. ` + + `Untrusted execution on a non-Kubernetes environment is refused.`, + deniedDriver: candidate.driver, + deniedProvider: provider, + }; +} diff --git a/server/src/services/execution-policy-bootstrap.test.ts b/server/src/services/execution-policy-bootstrap.test.ts new file mode 100644 index 00000000..21f197e0 --- /dev/null +++ b/server/src/services/execution-policy-bootstrap.test.ts @@ -0,0 +1,168 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const updateGeneral = vi.fn(); +const listCompanyIds = vi.fn(); +const ensureKubernetesEnvironment = vi.fn(); + +vi.mock("./instance-settings.js", () => ({ + instanceSettingsService: () => ({ + updateGeneral, + listCompanyIds, + }), +})); + +vi.mock("./environments.js", () => ({ + environmentService: () => ({ + ensureKubernetesEnvironment, + }), +})); + +const { + parseExecutionPolicyBootstrapEnv, + applyExecutionPolicyBootstrap, +} = await import("./execution-policy-bootstrap.js"); +type ExecutionPolicyBootstrapEnv = import("./execution-policy-bootstrap.js").ExecutionPolicyBootstrapEnv; +type ExecutionPolicyBootstrap = import("./execution-policy-bootstrap.js").ExecutionPolicyBootstrap; + +function env(overrides: Record): ExecutionPolicyBootstrapEnv { + return overrides; +} + +const bootstrap: ExecutionPolicyBootstrap = { + executionMode: "kubernetes", + kubernetesConfig: { inCluster: true, backend: "job" }, +}; + +// `applyExecutionPolicyBootstrap` constructs its services internally from the +// Db, so we mock the service modules; the Db itself is never touched here. +const fakeDb = {} as never; + +describe("parseExecutionPolicyBootstrapEnv", () => { + it("returns null when no execution mode is set (default unrestricted)", () => { + expect(parseExecutionPolicyBootstrapEnv(env({}))).toBeNull(); + }); + + it("returns null when execution mode is explicitly any", () => { + expect( + parseExecutionPolicyBootstrapEnv(env({ PAPERCLIP_EXECUTION_MODE: "any" })), + ).toBeNull(); + }); + + it("parses the forced kubernetes policy with a job/gvisor/cilium config", () => { + const parsed = parseExecutionPolicyBootstrapEnv( + env({ + PAPERCLIP_EXECUTION_MODE: "kubernetes", + PAPERCLIP_K8S_BACKEND: "job", + PAPERCLIP_K8S_IN_CLUSTER: "true", + PAPERCLIP_K8S_RUNTIME_CLASS_NAME: "gvisor", + PAPERCLIP_K8S_EGRESS_MODE: "cilium", + PAPERCLIP_K8S_EGRESS_ALLOW_FQDNS: "api.anthropic.com, api.openai.com", + PAPERCLIP_K8S_EGRESS_ALLOW_CIDRS: "10.0.0.0/8", + }), + ); + expect(parsed).not.toBeNull(); + expect(parsed?.executionMode).toBe("kubernetes"); + expect(parsed?.kubernetesConfig).toMatchObject({ + backend: "job", + inCluster: true, + runtimeClassName: "gvisor", + egressMode: "cilium", + egressAllowFqdns: ["api.anthropic.com", "api.openai.com"], + egressAllowCidrs: ["10.0.0.0/8"], + }); + }); + + it("defaults inCluster false and omits unset optional fields", () => { + const parsed = parseExecutionPolicyBootstrapEnv( + env({ PAPERCLIP_EXECUTION_MODE: "kubernetes" }), + ); + expect(parsed?.kubernetesConfig.inCluster).toBe(false); + expect(parsed?.kubernetesConfig.runtimeClassName).toBeUndefined(); + expect(parsed?.kubernetesConfig.egressAllowFqdns).toBeUndefined(); + }); + + it("throws on an unknown execution mode", () => { + expect(() => + parseExecutionPolicyBootstrapEnv(env({ PAPERCLIP_EXECUTION_MODE: "vm" })), + ).toThrow(/PAPERCLIP_EXECUTION_MODE/); + }); + + it("attaches the declared adapter registry to the kubernetes config", () => { + const parsed = parseExecutionPolicyBootstrapEnv( + env({ + PAPERCLIP_EXECUTION_MODE: "kubernetes", + PAPERCLIP_ADAPTERS: JSON.stringify([ + { adapterType: "opencode_local", runtimeImage: "img", envKeys: ["ANTHROPIC_API_KEY"], allowFqdns: [], probeCommand: ["opencode", "--version"], defaultEnv: { ANTHROPIC_BASE_URL: "http://bifrost:8080" } }, + ]), + }), + ); + expect(parsed?.kubernetesConfig.adapters).toHaveLength(1); + expect(parsed?.kubernetesConfig.adapters?.[0].adapterType).toBe("opencode_local"); + }); + + it("leaves adapters undefined when PAPERCLIP_ADAPTERS is absent", () => { + const parsed = parseExecutionPolicyBootstrapEnv(env({ PAPERCLIP_EXECUTION_MODE: "kubernetes" })); + expect(parsed?.kubernetesConfig.adapters).toBeUndefined(); + }); + + it("reads PAPERCLIP_K8S_RPC_TIMEOUT_MS into kubernetesConfig.timeoutMs", () => { + const parsed = parseExecutionPolicyBootstrapEnv( + env({ + PAPERCLIP_EXECUTION_MODE: "kubernetes", + PAPERCLIP_K8S_RPC_TIMEOUT_MS: "600000", + }), + ); + expect(parsed?.kubernetesConfig.timeoutMs).toBe(600000); + }); + + it("omits timeoutMs when PAPERCLIP_K8S_RPC_TIMEOUT_MS is absent", () => { + const parsed = parseExecutionPolicyBootstrapEnv(env({ PAPERCLIP_EXECUTION_MODE: "kubernetes" })); + expect(parsed?.kubernetesConfig.timeoutMs).toBeUndefined(); + }); + + it("throws when PAPERCLIP_K8S_RPC_TIMEOUT_MS is not a positive integer", () => { + expect(() => + parseExecutionPolicyBootstrapEnv( + env({ PAPERCLIP_EXECUTION_MODE: "kubernetes", PAPERCLIP_K8S_RPC_TIMEOUT_MS: "0" }), + ), + ).toThrow(/PAPERCLIP_K8S_RPC_TIMEOUT_MS/); + expect(() => + parseExecutionPolicyBootstrapEnv( + env({ PAPERCLIP_EXECUTION_MODE: "kubernetes", PAPERCLIP_K8S_RPC_TIMEOUT_MS: "abc" }), + ), + ).toThrow(/PAPERCLIP_K8S_RPC_TIMEOUT_MS/); + }); +}); + +describe("applyExecutionPolicyBootstrap", () => { + beforeEach(() => { + updateGeneral.mockReset().mockResolvedValue(undefined); + listCompanyIds.mockReset(); + ensureKubernetesEnvironment.mockReset(); + }); + + it("does not throw when every company gets a managed environment", async () => { + listCompanyIds.mockResolvedValue(["c1", "c2", "c3"]); + ensureKubernetesEnvironment.mockResolvedValue({ id: "env" }); + + const result = await applyExecutionPolicyBootstrap(fakeDb, bootstrap); + + expect(result).toEqual({ executionMode: "kubernetes", companiesConfigured: 3 }); + expect(ensureKubernetesEnvironment).toHaveBeenCalledTimes(3); + }); + + it("throws when at least one company fails, after attempting every company", async () => { + listCompanyIds.mockResolvedValue(["c1", "c2", "c3"]); + ensureKubernetesEnvironment.mockImplementation(async (companyId: string) => { + if (companyId === "c2") throw new Error("operator config missing"); + return { id: `env-${companyId}` }; + }); + + await expect(applyExecutionPolicyBootstrap(fakeDb, bootstrap)).rejects.toThrow( + /execution-policy bootstrap: 1 of 3 companies failed.*c2/, + ); + + // It keeps going past the failure (attempts all three companies). + expect(ensureKubernetesEnvironment).toHaveBeenCalledTimes(3); + }); +}); diff --git a/server/src/services/execution-policy-bootstrap.ts b/server/src/services/execution-policy-bootstrap.ts new file mode 100644 index 00000000..a8428843 --- /dev/null +++ b/server/src/services/execution-policy-bootstrap.ts @@ -0,0 +1,194 @@ +/** + * Cloud execution-policy bootstrap. + * + * Lets an operator / gitops deployment force the instance onto the Kubernetes + * sandbox provider purely via environment variables, with no manual product-API + * calls. On startup we: + * 1. Parse `PAPERCLIP_EXECUTION_MODE` (+ `PAPERCLIP_K8S_*`) from the env. + * 2. Persist `executionMode` into instance general settings (so the per-run + * heartbeat guard enforces it). + * 3. Idempotently ensure a configured Kubernetes sandbox environment for every + * company (mirrors `ensureLocalEnvironment`). + * + * The boot hook is *configuration convenience*; the actual security gate is the + * per-run guard in the heartbeat (see `execution-allowlist.ts`). Even with no + * boot hook, setting `executionMode=kubernetes` denies local execution. + * + * The env-var parsing is a pure function so it is trivially unit-testable. + */ + +import type { Db } from "@paperclipai/db"; +import type { InstanceExecutionMode } from "@paperclipai/shared"; +import { logger } from "../middleware/logger.js"; +import { environmentService, type KubernetesEnvironmentConfigInput } from "./environments.js"; +import { instanceSettingsService } from "./instance-settings.js"; +import { parseAdapterRegistryEnv } from "./adapter-registry-bootstrap.js"; + +export type ExecutionPolicyBootstrapEnv = Record; + +export interface ExecutionPolicyBootstrap { + executionMode: Extract; + kubernetesConfig: KubernetesEnvironmentConfigInput; +} + +function parseBool(value: string | undefined): boolean | undefined { + if (value === undefined) return undefined; + const v = value.trim().toLowerCase(); + if (v === "true" || v === "1" || v === "yes") return true; + if (v === "false" || v === "0" || v === "no") return false; + return undefined; +} + +function parsePositiveIntMs(value: string | undefined): number | undefined { + if (value === undefined) return undefined; + const trimmed = value.trim(); + if (trimmed.length === 0) return undefined; + const parsed = Number(trimmed); + if (!Number.isFinite(parsed) || !Number.isInteger(parsed) || parsed <= 0) { + throw new Error( + `PAPERCLIP_K8S_RPC_TIMEOUT_MS must be a positive integer of milliseconds (got "${value}").`, + ); + } + return parsed; +} + +function parseList(value: string | undefined): string[] | undefined { + if (value === undefined) return undefined; + const items = value + .split(",") + .map((s) => s.trim()) + .filter((s) => s.length > 0); + return items.length > 0 ? items : undefined; +} + +/** + * Parse the forced-execution-mode env config. Returns null when execution is + * unrestricted (no env, or `PAPERCLIP_EXECUTION_MODE=any`). Throws on an + * unrecognized mode so a misconfigured deployment fails loudly instead of + * silently allowing local execution. + */ +export function parseExecutionPolicyBootstrapEnv( + env: ExecutionPolicyBootstrapEnv, +): ExecutionPolicyBootstrap | null { + const raw = env.PAPERCLIP_EXECUTION_MODE?.trim(); + if (!raw || raw === "any") return null; + if (raw !== "kubernetes") { + throw new Error( + `PAPERCLIP_EXECUTION_MODE must be "kubernetes" or "any" (got "${raw}").`, + ); + } + + const kubernetesConfig: KubernetesEnvironmentConfigInput = { + // inCluster defaults to false (matches the plugin schema default); an + // in-cluster cloud deployment sets PAPERCLIP_K8S_IN_CLUSTER=true. + inCluster: parseBool(env.PAPERCLIP_K8S_IN_CLUSTER) ?? false, + }; + + const backend = env.PAPERCLIP_K8S_BACKEND?.trim(); + if (backend) { + if (backend !== "job" && backend !== "sandbox-cr") { + throw new Error( + `PAPERCLIP_K8S_BACKEND must be "job" or "sandbox-cr" (got "${backend}").`, + ); + } + kubernetesConfig.backend = backend; + } + + const egressMode = env.PAPERCLIP_K8S_EGRESS_MODE?.trim(); + if (egressMode) { + if (egressMode !== "cilium" && egressMode !== "standard") { + throw new Error( + `PAPERCLIP_K8S_EGRESS_MODE must be "cilium" or "standard" (got "${egressMode}").`, + ); + } + kubernetesConfig.egressMode = egressMode; + } + + const runtimeClassName = env.PAPERCLIP_K8S_RUNTIME_CLASS_NAME?.trim(); + if (runtimeClassName) kubernetesConfig.runtimeClassName = runtimeClassName; + + const namespacePrefix = env.PAPERCLIP_K8S_NAMESPACE_PREFIX?.trim(); + if (namespacePrefix) kubernetesConfig.namespacePrefix = namespacePrefix; + + const imageRegistry = env.PAPERCLIP_K8S_IMAGE_REGISTRY?.trim(); + if (imageRegistry) kubernetesConfig.imageRegistry = imageRegistry; + + const rpcTimeoutMs = parsePositiveIntMs(env.PAPERCLIP_K8S_RPC_TIMEOUT_MS); + if (rpcTimeoutMs !== undefined) kubernetesConfig.timeoutMs = rpcTimeoutMs; + + const adapterType = env.PAPERCLIP_K8S_ADAPTER_TYPE?.trim(); + if (adapterType) kubernetesConfig.adapterType = adapterType; + + const egressAllowFqdns = parseList(env.PAPERCLIP_K8S_EGRESS_ALLOW_FQDNS); + if (egressAllowFqdns) kubernetesConfig.egressAllowFqdns = egressAllowFqdns; + + const egressAllowCidrs = parseList(env.PAPERCLIP_K8S_EGRESS_ALLOW_CIDRS); + if (egressAllowCidrs) kubernetesConfig.egressAllowCidrs = egressAllowCidrs; + + const adapters = parseAdapterRegistryEnv(env); + if (adapters) kubernetesConfig.adapters = adapters; + + return { executionMode: "kubernetes", kubernetesConfig }; +} + +/** + * Apply the parsed bootstrap to the database: persist `executionMode` into + * instance settings and ensure a configured Kubernetes environment for every + * company. Idempotent; safe to call on every boot. + */ +export async function applyExecutionPolicyBootstrap( + db: Db, + bootstrap: ExecutionPolicyBootstrap, +): Promise<{ executionMode: InstanceExecutionMode; companiesConfigured: number }> { + const instanceSettings = instanceSettingsService(db); + const environments = environmentService(db); + + await instanceSettings.updateGeneral({ executionMode: bootstrap.executionMode }); + + const companyIds = await instanceSettings.listCompanyIds(); + let configured = 0; + const failedCompanyIds: string[] = []; + for (const companyId of companyIds) { + try { + await environments.ensureKubernetesEnvironment(companyId, bootstrap.kubernetesConfig); + configured += 1; + } catch (err) { + logger.error( + { err, companyId }, + "failed to ensure managed Kubernetes environment during execution-policy bootstrap", + ); + failedCompanyIds.push(companyId); + } + } + + logger.info( + { + executionMode: bootstrap.executionMode, + companiesConfigured: configured, + backend: bootstrap.kubernetesConfig.backend, + runtimeClassName: bootstrap.kubernetesConfig.runtimeClassName, + egressMode: bootstrap.kubernetesConfig.egressMode, + }, + "applied forced Kubernetes execution policy", + ); + + if (failedCompanyIds.length > 0) { + throw new Error( + `execution-policy bootstrap: ${failedCompanyIds.length} of ${companyIds.length} companies failed to get a managed Kubernetes environment under executionMode=${bootstrap.executionMode}; refusing to start (companies: ${failedCompanyIds.join(", ")})`, + ); + } + + return { executionMode: bootstrap.executionMode, companiesConfigured: configured }; +} + +/** + * Convenience: parse + apply from a raw env map. Returns null when unrestricted. + */ +export async function bootstrapExecutionPolicyFromEnv( + db: Db, + env: ExecutionPolicyBootstrapEnv = process.env, +): Promise<{ executionMode: InstanceExecutionMode; companiesConfigured: number } | null> { + const bootstrap = parseExecutionPolicyBootstrapEnv(env); + if (!bootstrap) return null; + return applyExecutionPolicyBootstrap(db, bootstrap); +} diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index b88d6995..deec2ab6 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -132,6 +132,10 @@ import { resolveExecutionWorkspaceMode, } from "./execution-workspace-policy.js"; import { instanceSettingsService } from "./instance-settings.js"; +import { + evaluateExecutionAllowlist, + isExecutionForcedToKubernetes, +} from "./execution-allowlist.js"; import { RECOVERY_ORIGIN_KINDS, FINISH_SUCCESSFUL_RUN_HANDOFF_REASON, @@ -182,6 +186,7 @@ import { } from "@paperclipai/adapter-utils/server-utils"; import { extractSkillMentionIds, isUuidLike } from "@paperclipai/shared"; import { environmentService } from "./environments.js"; +import { parseExecutionPolicyBootstrapEnv } from "./execution-policy-bootstrap.js"; import { environmentRuntimeService } from "./environment-runtime.js"; import { environmentRunOrchestrator } from "./environment-run-orchestrator.js"; import { isUnsafeSessionWorkspaceCwd } from "./session-workspace-cwd.js"; @@ -8081,7 +8086,72 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) persistedExecutionWorkspaceMode === "agent_default" ? persistedExecutionWorkspaceMode : requestedExecutionWorkspaceMode; - const selectedEnvironmentId = environmentResolution.environmentId; + const executionPolicy = { executionMode: (await instanceSettings.getGeneral()).executionMode }; + let selectedEnvironmentId = environmentResolution.environmentId; + if (isExecutionForcedToKubernetes(executionPolicy)) { + let kubernetesEnvironment = await environmentsSvc.findKubernetesEnvironment(agent.companyId); + if (!kubernetesEnvironment) { + // Lazy recovery for companies created after the startup bootstrap ran + // (the boot hook only provisions environments for companies that exist + // at boot). Re-derive the managed-env config from the bootstrap env. + // If the process env no longer forces Kubernetes (rollback / config + // drift relative to the persisted executionMode setting), skip the + // provisioning gracefully: the guard below still refuses local + // fallback with the explicit error, instead of crashing here on + // undefined config. + let bootstrap: ReturnType = null; + let bootstrapSkipReason: string | null = null; + try { + bootstrap = parseExecutionPolicyBootstrapEnv(process.env); + if (!bootstrap) { + bootstrapSkipReason = + 'PAPERCLIP_EXECUTION_MODE bootstrap env is not kubernetes-forced (absent or "any")'; + } + } catch (err) { + bootstrapSkipReason = `PAPERCLIP_EXECUTION_MODE bootstrap env failed to parse: ${ + err instanceof Error ? err.message : String(err) + }`; + } + if (bootstrap) { + await environmentsSvc.ensureKubernetesEnvironment( + agent.companyId, + bootstrap.kubernetesConfig, + ); + kubernetesEnvironment = await environmentsSvc.findKubernetesEnvironment(agent.companyId); + } else { + logger.warn( + { + runId: run.id, + agentId: agent.id, + companyId: agent.companyId, + reason: bootstrapSkipReason, + }, + "executionMode=kubernetes is persisted but the bootstrap env cannot provision a managed Kubernetes environment; skipping lazy provisioning for this company (the run will fail with the explicit no-managed-environment error)", + ); + } + } + if (!kubernetesEnvironment) { + throw new Error( + "Instance execution policy requires the Kubernetes sandbox provider " + + "(executionMode=kubernetes) but no managed Kubernetes environment is " + + "configured for this company. Configure one (PAPERCLIP_K8S_* env on the " + + "cloud instance) before running agents; refusing to fall back to local execution.", + ); + } + if (kubernetesEnvironment.id !== selectedEnvironmentId) { + logger.info( + { + runId: run.id, + issueId, + agentId: agent.id, + resolvedEnvironmentId: selectedEnvironmentId, + forcedKubernetesEnvironmentId: kubernetesEnvironment.id, + }, + "Forcing run onto the managed Kubernetes environment (executionMode=kubernetes)", + ); + } + selectedEnvironmentId = kubernetesEnvironment.id; + } const { selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver, workspace: resolvedWorkspace, @@ -8402,7 +8472,13 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }) .where(eq(heartbeatRuns.id, run.id)); } - const persistedEnvironmentId = persistedExecutionWorkspace?.config?.environmentId ?? selectedEnvironmentId; + // When execution is forced to Kubernetes, `selectedEnvironmentId` is already + // pinned to the managed k8s environment above; ignore any persisted workspace + // environmentId (which could point at a stale local/ssh env) so a reused + // workspace can never downgrade us off the sandbox. + const persistedEnvironmentId = isExecutionForcedToKubernetes(executionPolicy) + ? selectedEnvironmentId + : persistedExecutionWorkspace?.config?.environmentId ?? selectedEnvironmentId; const acquiredEnvironment = await envOrchestrator.acquireForRun({ companyId: agent.companyId, selectedEnvironmentId: persistedEnvironmentId, @@ -8414,6 +8490,30 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) persistedExecutionWorkspace, }); const selectedEnvironment = acquiredEnvironment.environment; + // Defense-in-depth: re-check the actually-acquired environment against the + // execution allowlist. Even if selection were bypassed, a denied (local/ssh/ + // non-k8s) environment FAILS the run here rather than executing untrusted. + const allowlistDecision = evaluateExecutionAllowlist(executionPolicy, { + driver: selectedEnvironment.driver, + provider: + typeof selectedEnvironment.config?.provider === "string" + ? selectedEnvironment.config.provider + : null, + }); + if (!allowlistDecision.allowed) { + logger.error( + { + runId: run.id, + issueId, + agentId: agent.id, + environmentId: selectedEnvironment.id, + deniedDriver: allowlistDecision.deniedDriver, + deniedProvider: allowlistDecision.deniedProvider, + }, + "Execution allowlist denied the resolved environment; failing run", + ); + throw new Error(allowlistDecision.reason); + } let activeEnvironmentLease = { environment: acquiredEnvironment.environment, lease: acquiredEnvironment.lease, diff --git a/server/src/services/index.ts b/server/src/services/index.ts index 7affa2e9..c299687c 100644 --- a/server/src/services/index.ts +++ b/server/src/services/index.ts @@ -62,6 +62,7 @@ export type { } from "./authorization.js"; export { boardAuthService } from "./board-auth.js"; export { instanceSettingsService } from "./instance-settings.js"; +export { bootstrapExecutionPolicyFromEnv } from "./execution-policy-bootstrap.js"; export { cloudUpstreamService, reconcileCloudUpstreamRunsOnStartup } from "./cloud-upstreams.js"; export { companyPortabilityService } from "./company-portability.js"; export { teamsCatalogService } from "./teams-catalog.js"; diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index 170c8ca2..22b71466 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -27,6 +27,8 @@ function normalizeGeneralSettings(raw: unknown): InstanceGeneralSettings { feedbackDataSharingPreference: parsed.data.feedbackDataSharingPreference ?? DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE, backupRetention: parsed.data.backupRetention ?? DEFAULT_BACKUP_RETENTION, + // Absent => unrestricted; only carry through an explicit policy. + ...(parsed.data.executionMode ? { executionMode: parsed.data.executionMode } : {}), }; } return { diff --git a/server/src/services/plugin-loader.ts b/server/src/services/plugin-loader.ts index 8d7c10bc..0424f0b1 100644 --- a/server/src/services/plugin-loader.ts +++ b/server/src/services/plugin-loader.ts @@ -79,6 +79,16 @@ export const DEFAULT_LOCAL_PLUGIN_DIR = path.join( const DEV_TSX_LOADER_PATH = path.resolve(__dirname, "../../../cli/node_modules/tsx/dist/loader.mjs"); +/** + * Model-provider API keys that sandbox-provider plugins (e.g. + * `@paperclipai/plugin-kubernetes`) are allowed to read from the + * server's process environment so they can inject them into per-run + * pod Secrets. All other host env vars remain stripped from plugin + * workers (see `PluginWorkerManager.spawnProcess`). The passthrough + * is gated on the plugin manifest declaring + * `environment.drivers.register` — non-sandbox plugins never receive + * these keys. + */ const ADAPTER_ENV_PASSTHROUGH = [ "ANTHROPIC_API_KEY", "OPENAI_API_KEY", @@ -87,6 +97,21 @@ const ADAPTER_ENV_PASSTHROUGH = [ "OPENROUTER_API_KEY", ]; +/** + * In-cluster Kubernetes service-discovery vars. A sandbox-provider plugin that + * runs in-cluster (e.g. `@paperclipai/plugin-kubernetes` with inCluster=true) + * builds its API client via `KubeConfig.loadFromCluster()`, which reads these + * to construct the apiserver URL. Without them the worker fails with "Invalid + * URL" at lease acquisition. The CA + token are files under + * /var/run/secrets/kubernetes.io/serviceaccount and are readable directly. + * Gated, like ADAPTER_ENV_PASSTHROUGH, on environment-driver registration. + */ +const K8S_IN_CLUSTER_ENV_PASSTHROUGH = [ + "KUBERNETES_SERVICE_HOST", + "KUBERNETES_SERVICE_PORT", + "KUBERNETES_SERVICE_PORT_HTTPS", +]; + export function buildPluginWorkerEnv(input: { manifest: Pick; instanceInfo: { deploymentMode?: string | null; deploymentExposure?: string | null }; @@ -101,7 +126,7 @@ export function buildPluginWorkerEnv(input: { && input.manifest.capabilities.includes("environment.drivers.register"); if (!canRegisterEnvironmentDrivers) return env; - for (const key of ADAPTER_ENV_PASSTHROUGH) { + for (const key of [...ADAPTER_ENV_PASSTHROUGH, ...K8S_IN_CLUSTER_ENV_PASSTHROUGH]) { const value = processEnv[key]; if (value && value.trim().length > 0) { env[key] = value; diff --git a/ui/src/components/AgentConfigForm.test.ts b/ui/src/components/AgentConfigForm.test.ts index e532c8eb..09905880 100644 --- a/ui/src/components/AgentConfigForm.test.ts +++ b/ui/src/components/AgentConfigForm.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it } from "vitest"; +import type { Environment } from "@paperclipai/shared"; import { supportsAdapterModelRefresh } from "./AgentConfigForm"; +import { resolveForcedKubernetesEnvironment } from "../lib/forced-kubernetes-environment"; describe("supportsAdapterModelRefresh", () => { it("enables the model refresh action for Claude, Codex, and ACPX adapters", () => { @@ -13,3 +15,58 @@ describe("supportsAdapterModelRefresh", () => { expect(supportsAdapterModelRefresh("process")).toBe(false); }); }); + +function makeEnvironment(overrides: Partial): Environment { + return { + id: "env-1", + companyId: "co-1", + name: "Env", + description: null, + driver: "local", + status: "active", + config: {}, + metadata: null, + createdAt: new Date(0), + updatedAt: new Date(0), + ...overrides, + }; +} + +const localEnv = makeEnvironment({ id: "local-1", name: "Local", driver: "local" }); +const k8sEnv = makeEnvironment({ + id: "k8s-1", + name: "Managed K8s", + driver: "sandbox", + config: { provider: "kubernetes" }, +}); + +describe("resolveForcedKubernetesEnvironment", () => { + it("does not force when executionMode is 'any' (full picker / unchanged)", () => { + const result = resolveForcedKubernetesEnvironment("any", [localEnv, k8sEnv]); + expect(result.forced).toBe(false); + expect(result.kubernetesEnvironment).toBeNull(); + }); + + it("does not force when executionMode is absent (self-hoster default)", () => { + const result = resolveForcedKubernetesEnvironment(undefined, [localEnv, k8sEnv]); + expect(result.forced).toBe(false); + expect(result.kubernetesEnvironment).toBeNull(); + }); + + it("forces and selects the Kubernetes sandbox when executionMode is 'kubernetes'", () => { + const result = resolveForcedKubernetesEnvironment("kubernetes", [localEnv, k8sEnv]); + expect(result.forced).toBe(true); + expect(result.kubernetesEnvironment?.id).toBe("k8s-1"); + }); + + it("forces but reports no environment when none is the Kubernetes sandbox", () => { + const fakeSandbox = makeEnvironment({ + id: "fake-1", + driver: "sandbox", + config: { provider: "fake" }, + }); + const result = resolveForcedKubernetesEnvironment("kubernetes", [localEnv, fakeSandbox]); + expect(result.forced).toBe(true); + expect(result.kubernetesEnvironment).toBeNull(); + }); +}); diff --git a/ui/src/components/AgentConfigForm.tsx b/ui/src/components/AgentConfigForm.tsx index 264e61e9..342a7583 100644 --- a/ui/src/components/AgentConfigForm.tsx +++ b/ui/src/components/AgentConfigForm.tsx @@ -55,6 +55,7 @@ import { useDisabledAdaptersSync } from "../adapters/use-disabled-adapters"; import { buildAgentUpdatePatch, type AgentConfigOverlay } from "../lib/agent-config-patch"; import { useAdapterCapabilities } from "../adapters/use-adapter-capabilities"; import { filterAcpxModelsByAgent } from "../lib/acpx-model-filter"; +import { resolveForcedKubernetesEnvironment } from "../lib/forced-kubernetes-environment"; /* ---- Create mode values ---- */ @@ -223,11 +224,33 @@ export function AgentConfigForm(props: AgentConfigFormProps) { }); const environmentsEnabled = experimentalSettings?.enableEnvironments === true; + // Instance execution policy (general settings). When `executionMode` is + // "kubernetes" the instance FORCES all execution onto the managed Kubernetes + // sandbox; "any"/absent leaves the full environment/adapter choice intact. + // Reuses the same general-settings query the rest of the UI uses. + const { data: generalSettings } = useQuery({ + queryKey: queryKeys.instance.generalSettings, + queryFn: () => instanceSettingsApi.getGeneral(), + retry: false, + }); + const { data: environments = [] } = useQuery({ queryKey: selectedCompanyId ? queryKeys.environments.list(selectedCompanyId) : ["environments", "none"], queryFn: () => environmentsApi.list(selectedCompanyId!), - enabled: Boolean(selectedCompanyId) && environmentsEnabled, + // Load environments when the picker is enabled OR when execution is forced + // onto Kubernetes (so we can resolve and default to the managed K8s env even + // when the experimental environments picker is otherwise hidden). + enabled: + Boolean(selectedCompanyId) && + (environmentsEnabled || generalSettings?.executionMode === "kubernetes"), }); + + // Setting-driven: resolve whether the instance forces Kubernetes execution and + // which loaded environment is the managed Kubernetes sandbox. + const { forced: forcedKubernetes, kubernetesEnvironment } = useMemo( + () => resolveForcedKubernetesEnvironment(generalSettings?.executionMode, environments), + [generalSettings?.executionMode, environments], + ); const createSecret = useMutation({ mutationFn: (input: { name: string; value: string }) => { if (!selectedCompanyId) throw new Error("Select a company to create secrets"); @@ -336,6 +359,17 @@ export function AgentConfigForm(props: AgentConfigFormProps) { () => environments.find((environment) => environment.id === currentDefaultEnvironmentId) ?? null, [currentDefaultEnvironmentId, environments], ); + + // When the instance forces Kubernetes execution, new agents must default to the + // managed Kubernetes sandbox environment (never the implicit local default). + // Only applies in create mode and only once the K8s environment is loaded; if + // none is available the UI surfaces a notice instead of silently selecting it. + useEffect(() => { + if (!isCreate || !set || !forcedKubernetes || !kubernetesEnvironment) return; + if (currentDefaultEnvironmentId === kubernetesEnvironment.id) return; + set({ defaultEnvironmentId: kubernetesEnvironment.id }); + }, [isCreate, set, forcedKubernetes, kubernetesEnvironment, currentDefaultEnvironmentId]); + const runnableEnvironments = useMemo( () => environments.filter((environment) => { if (!supportedEnvironmentDrivers.has(environment.driver)) return false; @@ -782,7 +816,35 @@ export function AgentConfigForm(props: AgentConfigFormProps) { )} {/* ---- Execution ---- */} - {environmentsEnabled ? ( + {forcedKubernetes ? ( + // Instance execution policy forces the managed Kubernetes sandbox + // (executionMode=kubernetes): never offer local / non-Kubernetes targets. + // Render the environment read-only instead of the selectable picker. +
+ {cards + ?

Execution

+ :
Execution
+ } +
+ + {kubernetesEnvironment ? ( +
+ {kubernetesEnvironment.name} · Kubernetes sandbox +
+ ) : ( +
+ This instance requires the Kubernetes sandbox, but no managed Kubernetes + environment is available for this company yet. Configure one before creating + agents; execution will not fall back to local. +
+ )} +
+
+
+ ) : environmentsEnabled ? (
{cards ?

Execution

diff --git a/ui/src/lib/forced-kubernetes-environment.ts b/ui/src/lib/forced-kubernetes-environment.ts new file mode 100644 index 00000000..61ffd477 --- /dev/null +++ b/ui/src/lib/forced-kubernetes-environment.ts @@ -0,0 +1,56 @@ +import type { Environment, InstanceExecutionMode } from "@paperclipai/shared"; + +/** + * Provider key (== plugin driverKey) of the first-party Kubernetes sandbox + * provider. Mirrors `KUBERNETES_PROVIDER_KEY` in the server-side execution + * allowlist. Kept local to the UI because the allowlist module lives in the + * server package and must not be imported by the client bundle. + */ +export const KUBERNETES_PROVIDER_KEY = "kubernetes" as const; + +/** + * True iff the environment is the Kubernetes sandbox provider, i.e. a core + * `sandbox` driver whose `config.provider` is "kubernetes". Mirrors the + * server-side `isKubernetesSandboxEnvironment` guard so the UI selects exactly + * the environment the server forces execution onto. + */ +export function isKubernetesSandboxEnvironment(environment: Environment): boolean { + if (environment.driver !== "sandbox") return false; + const provider = environment.config?.provider; + return provider === KUBERNETES_PROVIDER_KEY; +} + +export interface ForcedKubernetesEnvironment { + /** + * Whether the instance execution policy forces all execution onto the + * Kubernetes sandbox. Driven entirely by the `executionMode` instance general + * setting: `"kubernetes"` forces; `"any"`/absent does not. A self-hoster who + * keeps the default `"any"` retains the full environment/adapter picker. + */ + forced: boolean; + /** + * The company's managed Kubernetes sandbox environment, if one is present in + * the loaded environment list. `null` when forced but no such environment is + * available yet (the UI should show a clear notice rather than silently + * defaulting to local). + */ + kubernetesEnvironment: Environment | null; +} + +/** + * Resolve whether execution is forced onto Kubernetes and, if so, which loaded + * environment is the Kubernetes sandbox. Pure so it can be unit-tested without + * rendering. + */ +export function resolveForcedKubernetesEnvironment( + executionMode: InstanceExecutionMode | undefined, + environments: readonly Environment[], +): ForcedKubernetesEnvironment { + const forced = executionMode === "kubernetes"; + if (!forced) { + return { forced: false, kubernetesEnvironment: null }; + } + const kubernetesEnvironment = + environments.find((environment) => isKubernetesSandboxEnvironment(environment)) ?? null; + return { forced: true, kubernetesEnvironment }; +} diff --git a/ui/storybook/stories/agent-management.stories.tsx b/ui/storybook/stories/agent-management.stories.tsx index 095b624c..b40512c9 100644 --- a/ui/storybook/stories/agent-management.stories.tsx +++ b/ui/storybook/stories/agent-management.stories.tsx @@ -8,6 +8,7 @@ import { type AgentRuntimeState, type CompanySecret, type EnvBinding, + type Environment, } from "@paperclipai/shared"; import { ActiveAgentsPanel } from "@/components/ActiveAgentsPanel"; import { AgentConfigForm, type CreateConfigValues } from "@/components/AgentConfigForm"; @@ -774,3 +775,88 @@ export default meta; type Story = StoryObj; export const ManagementMatrix: Story = {}; + +/* ---- Forced Kubernetes execution (instance executionMode=kubernetes) ---- */ + +const managedKubernetesEnvironment: Environment = { + id: "env-k8s-storybook", + companyId: COMPANY_ID, + name: "Kubernetes Sandbox", + description: "Managed Kubernetes sandbox environment for hosted tenant execution.", + driver: "sandbox", + status: "active", + config: { + provider: "kubernetes", + backend: "job", + inCluster: true, + runtimeClassName: "gvisor", + egressMode: "cilium", + }, + metadata: { managedByPaperclip: true, managedKubernetesSandbox: true }, + createdAt: recent(2_000), + updatedAt: recent(60), +}; + +function ForcedKubernetesFixtures({ + environmentFixtures, + children, +}: { + environmentFixtures: Environment[]; + children: ReactNode; +}) { + const queryClient = useQueryClient(); + + queryClient.setQueryData(queryKeys.agents.list(COMPANY_ID), agentManagementAgents); + queryClient.setQueryData(queryKeys.secrets.list(COMPANY_ID), storybookSecrets); + queryClient.setQueryData(queryKeys.adapters.all, adapterFixtures); + // The instance-level execution policy that forces all agent execution onto + // the managed Kubernetes sandbox (PAPERCLIP_EXECUTION_MODE=kubernetes). + queryClient.setQueryData(queryKeys.instance.generalSettings, { + censorUsernameInLogs: false, + executionMode: "kubernetes", + }); + queryClient.setQueryData(queryKeys.environments.list(COMPANY_ID), environmentFixtures); + queryClient.setQueryData(queryKeys.agents.adapterModels(COMPANY_ID, "codex_local"), [ + { id: "gpt-5.4", label: "GPT-5.4" }, + { id: "gpt-5.4-mini", label: "GPT-5.4 Mini" }, + ]); + + return children; +} + +function ForcedKubernetesStory({ environmentFixtures }: { environmentFixtures: Environment[] }) { + return ( + +
+
+
+
+ +
+
+
+
+
+ ); +} + +/** + * Instance execution policy forces Kubernetes and the company has a managed + * Kubernetes sandbox environment: the Execution section renders the + * environment read-only (no local/SSH picker). + */ +export const ForcedKubernetesExecution: Story = { + render: () => , +}; + +/** + * Instance execution policy forces Kubernetes but no managed environment is + * available for the company yet: the Execution section shows the warning + * notice instead of silently falling back to local execution. + */ +export const ForcedKubernetesMissingEnvironment: Story = { + render: () => , +};