Arnike/paperclip
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add referenced last30days catalog entry

Browse Source
This commit is contained in:
Dotta
2026-06-04 20:01:25 +00:00
parent 1227bb8ead
commit bee2b25f5d
17 changed files with 1276 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/__tests__/company-skills-catalog-service.test.ts
+41
View File
@@ -223,6 +223,47 @@ describeEmbeddedPostgres("companySkillService.installFromCatalog", () => {
});
});
it("installs script-bearing catalog skills when materialized bytes pass the static audit", async () => {
const script = "print('safe')\n";
const scriptFiles: CatalogSkillFile[] = [
...sampleFiles,
{ path: "scripts/run.py", kind: "script", sizeBytes: Buffer.byteLength(script), sha256: sha256(script) },
];
const scriptCatalogSkill: CatalogSkill = {
...sampleCatalogSkill,
trustLevel: "scripts_executables",
files: scriptFiles,
contentHash: contentHash(scriptFiles),
};
mockCatalogService.getCatalogSkillOrThrow.mockReturnValue(scriptCatalogSkill);
mockCatalogService.copyCatalogSkillFile.mockImplementation(async (_ref: string, filePath: string, targetPath: string) => {
if (filePath === "scripts/run.py") {
await fs.writeFile(targetPath, script, "utf8");
return;
}
const content = filePath === "SKILL.md" ? sampleSkillMarkdown : sampleReferenceMarkdown;
await fs.writeFile(targetPath, content, "utf8");
});
const companyId = await createCompany();
const result = await svc.installFromCatalog(companyId, {
catalogSkillId: scriptCatalogSkill.id,
});
expect(result.action).toBe("created");
expect(result.skill).toMatchObject({
trustLevel: "scripts_executables",
metadata: expect.objectContaining({
auditVerdict: "warning",
auditCodes: expect.arrayContaining(["script_trust"]),
}),
});
expect(result.warnings).toEqual(expect.arrayContaining([
"Skill includes a script file.",
]));
await expect(fs.readFile(path.join(result.skill.sourceLocator!, "scripts/run.py"), "utf8")).resolves.toBe(script);
});
it("restores portable catalog provenance when importing packaged skills", async () => {
const companyId = await createCompany();
const importedFiles = {
server/src/__tests__/skills-catalog-service.test.ts
+40
View File
@@ -1,3 +1,4 @@
import { createHash } from "node:crypto";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { CatalogSkill } from "@paperclipai/shared";
@@ -42,6 +43,10 @@ function catalogSkill(slug: string, name = slug): CatalogSkill {
};
}
function sha256(value: string | Buffer) {
return createHash("sha256").update(value).digest("hex");
}
function manifest(skills: CatalogSkill[], packageVersion = "0.3.1") {
return JSON.stringify({
schemaVersion: 1,
@@ -59,6 +64,7 @@ describe("skills catalog service", () => {
beforeEach(() => {
vi.resetModules();
vi.clearAllMocks();
vi.unstubAllGlobals();
manifestJson = manifest([catalogSkill("old-skill", "Old Skill")]);
manifestMtimeMs = 1;
mockExistsSync.mockReturnValue(true);
@@ -110,4 +116,38 @@ describe("skills catalog service", () => {
});
expect(mockReadFile).not.toHaveBeenCalled();
});
it("reads referenced GitHub catalog files from pinned raw bytes", async () => {
const markdown = "---\nname: remote\n---\n\n# Remote\n";
const remoteSkill = catalogSkill("remote-skill", "Remote Skill");
remoteSkill.source = {
type: "github",
hostname: "github.com",
owner: "example",
repo: "remote-skill",
ref: "v1.0.0",
commit: "0123456789abcdef0123456789abcdef01234567",
path: "skills/remote",
url: "https://github.com/example/remote-skill/tree/v1.0.0/skills/remote",
};
remoteSkill.files = [
{ path: "SKILL.md", kind: "skill", sizeBytes: Buffer.byteLength(markdown), sha256: sha256(markdown) },
];
manifestJson = manifest([remoteSkill]);
const fetchMock = vi.fn(async (url: string) => {
expect(url).toBe("https://raw.githubusercontent.com/example/remote-skill/0123456789abcdef0123456789abcdef01234567/skills/remote/SKILL.md");
return new Response(markdown, { status: 200 });
});
vi.stubGlobal("fetch", fetchMock);
const service = await import("../services/skills-catalog.js");
await expect(service.readCatalogSkillFile(remoteSkill.id, "SKILL.md")).resolves.toMatchObject({
catalogSkillId: remoteSkill.id,
path: "SKILL.md",
content: markdown,
markdown: true,
});
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(mockReadFile).not.toHaveBeenCalled();
});
});
server/src/services/company-skills.ts
+53 -10
View File
@@ -44,7 +44,6 @@ import {
copyCatalogSkillFile,
getCatalogPackageMetadata,
getCatalogSkillOrThrow,
readCatalogSkillFile,
resolveCatalogSkillReference,
} from "./skills-catalog.js";
import {
@@ -2452,7 +2451,7 @@ export function companySkillService(db: Db) {
buildSkillRuntimeName(catalogSkill.key, skill.slug),
);
await copySkillDirectory(originSnapshotLocator, materializedDir);
const markdown = (await readCatalogSkillFile(catalogSkill.id, catalogSkill.entrypoint)).content;
const markdown = await fs.readFile(path.join(originSnapshotLocator, catalogSkill.entrypoint), "utf8");
const nextMetadata = buildCatalogSkillMetadata(catalogSkill, skill, originSnapshotLocator);
const nextValues = {
name: catalogSkill.name,
@@ -2942,6 +2941,7 @@ export function companySkillService(db: Db) {
catalogKind: catalogSkill.kind,
catalogCategory: catalogSkill.category,
catalogPath: catalogSkill.path,
catalogSource: catalogSkill.source ?? null,
packageName: packageMetadata.packageName,
packageVersion: packageMetadata.packageVersion,
originHash: catalogSkill.contentHash,
@@ -2956,11 +2956,36 @@ export function companySkillService(db: Db) {
if (catalogSkill.compatibility !== "compatible") {
throw unprocessable(`Catalog skill ${catalogSkill.id} is not compatible.`);
}
if (catalogSkill.trustLevel === "scripts_executables") {
throw unprocessable(
"Catalog skill contains executable scripts and cannot be force-installed until security review semantics allow it.",
);
}
}
async function auditCatalogSkillSnapshot(
companyId: string,
catalogSkill: CatalogSkill,
slug: string,
sourceDir: string,
) {
return auditInstalledSkillBytes({
id: randomUUID(),
companyId,
key: catalogSkill.key,
slug,
name: catalogSkill.name,
description: catalogSkill.description,
markdown: "",
sourceType: "catalog",
sourceLocator: sourceDir,
sourceRef: catalogSkill.contentHash,
trustLevel: catalogSkill.trustLevel,
compatibility: catalogSkill.compatibility,
fileInventory: catalogSkill.files.map((entry) => ({ path: entry.path, kind: entry.kind })),
metadata: {
sourceKind: "catalog",
catalogId: catalogSkill.id,
originHash: catalogSkill.contentHash,
},
createdAt: new Date(),
updatedAt: new Date(),
});
}
async function installFromCatalog(
@@ -3021,9 +3046,27 @@ export function companySkillService(db: Db) {
}
}
const materializedDir = await materializeCatalogManifestSkillFiles(companyId, catalogSkill, slug);
const originSnapshotLocator = await materializeCatalogOriginSnapshot(companyId, catalogSkill, slug);
const markdown = (await readCatalogSkillFile(catalogSkill.id, catalogSkill.entrypoint)).content;
let materializedDir: string | null = null;
let originSnapshotLocator: string | null = null;
try {
originSnapshotLocator = await materializeCatalogOriginSnapshot(companyId, catalogSkill, slug);
const candidateAudit = await auditCatalogSkillSnapshot(companyId, catalogSkill, slug, originSnapshotLocator);
if (candidateAudit.verdict === "fail") {
throw unprocessable("Catalog install is blocked by hard-stop audit findings.", {
updateHoldReason: "audit_hard_stop",
audit: candidateAudit,
});
}
materializedDir = await materializeCatalogManifestSkillFiles(companyId, catalogSkill, slug);
} catch (error) {
if (materializedDir) await fs.rm(materializedDir, { recursive: true, force: true }).catch(() => undefined);
if (originSnapshotLocator) await fs.rm(originSnapshotLocator, { recursive: true, force: true }).catch(() => undefined);
throw error;
}
if (!materializedDir || !originSnapshotLocator) {
throw unprocessable("Catalog install did not materialize pinned files.");
}
const markdown = await fs.readFile(path.join(originSnapshotLocator, catalogSkill.entrypoint), "utf8");
const metadata = buildCatalogSkillMetadata(catalogSkill, existingByKey, originSnapshotLocator);
const values = {
companyId,
server/src/services/skills-catalog.ts
+60 -17
View File
@@ -1,3 +1,4 @@
import { createHash } from "node:crypto";
import { existsSync, readFileSync, statSync } from "node:fs";
import { promises as fs } from "node:fs";
import path from "node:path";
@@ -6,8 +7,10 @@ import type {
CatalogSkill,
CatalogSkillFileDetail,
CatalogSkillListQuery,
CatalogSkillSource,
} from "@paperclipai/shared";
import { HttpError, conflict, notFound } from "../errors.js";
import { HttpError, conflict, notFound, unprocessable } from "../errors.js";
import { ghFetch, resolveRawGitHubUrl } from "./github-fetch.js";
import { normalizePortablePath } from "./portable-path.js";
interface CatalogManifestFile {
@@ -93,6 +96,60 @@ function resolveCatalogPackageRoot() {
return catalogPackageRoot;
}
function sourceRootPath(source: CatalogSkillSource) {
return source.path ? normalizePortablePath(source.path) : "";
}
function resolveCatalogSourcePath(source: CatalogSkillSource, relativePath: string) {
const sourceRoot = sourceRootPath(source);
return sourceRoot ? `${sourceRoot}/${relativePath}` : relativePath;
}
async function fetchCatalogSourceFile(
skill: CatalogSkill,
relativePath: string,
): Promise<Buffer> {
const source = skill.source;
if (!source) {
const packageRoot = resolveCatalogPackageRoot();
const absolutePath = path.resolve(packageRoot, skill.path, relativePath);
const skillRoot = path.resolve(packageRoot, skill.path);
if (absolutePath !== skillRoot && !absolutePath.startsWith(`${skillRoot}${path.sep}`)) {
throw notFound("Catalog skill file not found");
}
return fs.readFile(absolutePath);
}
if (source.type !== "github") {
throw unprocessable(`Unsupported catalog source type: ${(source as { type: string }).type}`);
}
const sourcePath = resolveCatalogSourcePath(source, relativePath);
const url = resolveRawGitHubUrl(source.hostname, source.owner, source.repo, source.commit, sourcePath);
const response = await ghFetch(url);
if (!response.ok) {
throw unprocessable(`Failed to fetch pinned catalog file ${sourcePath} from ${source.owner}/${source.repo}@${source.commit}: HTTP ${response.status}`);
}
return Buffer.from(await response.arrayBuffer());
}
async function readCatalogFileBytes(
skill: CatalogSkill,
relativePath: string,
): Promise<Buffer> {
const fileEntry = skill.files.find((entry) => entry.path === relativePath);
if (!fileEntry) {
throw notFound("Catalog skill file not found");
}
const bytes = await fetchCatalogSourceFile(skill, relativePath);
const actualSha = createHash("sha256").update(bytes).digest("hex");
if (actualSha !== fileEntry.sha256) {
throw unprocessable(`Pinned catalog file hash mismatch for ${skill.id}:${relativePath}.`);
}
return bytes;
}
function searchText(skill: CatalogSkill) {
return [
skill.id,
@@ -152,18 +209,11 @@ export async function readCatalogSkillFile(
throw notFound("Catalog skill file not found");
}
const packageRoot = resolveCatalogPackageRoot();
const absolutePath = path.resolve(packageRoot, skill.path, normalizedPath);
const skillRoot = path.resolve(packageRoot, skill.path);
if (absolutePath !== skillRoot && !absolutePath.startsWith(`${skillRoot}${path.sep}`)) {
throw notFound("Catalog skill file not found");
}
if (fileEntry.kind === "asset") {
throw new HttpError(415, "Catalog asset previews are not supported.");
}
const content = await fs.readFile(absolutePath, "utf8");
const content = (await readCatalogFileBytes(skill, normalizedPath)).toString("utf8");
return {
catalogSkillId: skill.id,
path: normalizedPath,
@@ -182,14 +232,7 @@ export async function copyCatalogSkillFile(reference: string, relativePath: stri
throw notFound("Catalog skill file not found");
}
const packageRoot = resolveCatalogPackageRoot();
const absolutePath = path.resolve(packageRoot, skill.path, normalizedPath);
const skillRoot = path.resolve(packageRoot, skill.path);
if (absolutePath !== skillRoot && !absolutePath.startsWith(`${skillRoot}${path.sep}`)) {
throw notFound("Catalog skill file not found");
}
await fs.copyFile(absolutePath, targetPath);
await fs.writeFile(targetPath, await readCatalogFileBytes(skill, normalizedPath));
}
export function getCatalogPackageMetadata() {