Add referenced last30days catalog entry
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
buildCatalogManifest,
|
||||
formatCatalogManifest,
|
||||
@@ -13,6 +13,7 @@ const tempDirs: string[] = [];
|
||||
describe("skills catalog manifest", () => {
|
||||
afterEach(async () => {
|
||||
await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })));
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("builds stable manifest entries from catalog skill directories", async () => {
|
||||
@@ -59,6 +60,81 @@ describe("skills catalog manifest", () => {
|
||||
expect(result.manifest.skills[0]!.contentHash).toMatch(/^sha256:[a-f0-9]{64}$/);
|
||||
});
|
||||
|
||||
it("builds stable manifest entries from pinned GitHub references", async () => {
|
||||
const packageDir = await createCatalogPackage();
|
||||
const skillMarkdown = [
|
||||
"---",
|
||||
"name: Remote Research",
|
||||
"description: Research recent discussion from a pinned upstream skill.",
|
||||
"---",
|
||||
"",
|
||||
"Use this skill.",
|
||||
"",
|
||||
].join("\n");
|
||||
const script = "print('hello')\n";
|
||||
await writeReference(packageDir, "optional", "research", "remote-research", {
|
||||
source: {
|
||||
type: "github",
|
||||
hostname: "github.com",
|
||||
owner: "example",
|
||||
repo: "remote-skill",
|
||||
ref: "v1.0.0",
|
||||
commit: "0123456789abcdef0123456789abcdef01234567",
|
||||
path: "skills/remote-research",
|
||||
},
|
||||
files: ["SKILL.md", "scripts/**"],
|
||||
recommendedForRoles: ["researcher"],
|
||||
tags: ["research"],
|
||||
});
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
if (url.includes("/git/trees/")) {
|
||||
return new Response(JSON.stringify({
|
||||
tree: [
|
||||
{ path: "skills/remote-research/SKILL.md", type: "blob", size: Buffer.byteLength(skillMarkdown) },
|
||||
{ path: "skills/remote-research/scripts/run.py", type: "blob", size: Buffer.byteLength(script) },
|
||||
{ path: "README.md", type: "blob", size: 9 },
|
||||
],
|
||||
}), { status: 200 });
|
||||
}
|
||||
if (url.endsWith("/skills/remote-research/SKILL.md")) {
|
||||
return new Response(skillMarkdown, { status: 200 });
|
||||
}
|
||||
if (url.endsWith("/skills/remote-research/scripts/run.py")) {
|
||||
return new Response(script, { status: 200 });
|
||||
}
|
||||
return new Response("not found", { status: 404 });
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const result = await buildCatalogManifest({
|
||||
packageDir,
|
||||
generatedAt: "2026-05-26T00:00:00.000Z",
|
||||
});
|
||||
|
||||
expect(result.errors).toEqual([]);
|
||||
expect(result.manifest.skills[0]).toMatchObject({
|
||||
id: "paperclipai:optional:research:remote-research",
|
||||
key: "paperclipai/optional/research/remote-research",
|
||||
path: "catalog/optional/research/remote-research",
|
||||
trustLevel: "scripts_executables",
|
||||
recommendedForRoles: ["researcher"],
|
||||
tags: ["research"],
|
||||
source: {
|
||||
type: "github",
|
||||
owner: "example",
|
||||
repo: "remote-skill",
|
||||
ref: "v1.0.0",
|
||||
commit: "0123456789abcdef0123456789abcdef01234567",
|
||||
path: "skills/remote-research",
|
||||
},
|
||||
});
|
||||
expect(result.manifest.skills[0]!.files.map((file) => file.path)).toEqual([
|
||||
"SKILL.md",
|
||||
"scripts/run.py",
|
||||
]);
|
||||
expect(result.manifest.skills[0]!.contentHash).toMatch(/^sha256:[a-f0-9]{64}$/);
|
||||
});
|
||||
|
||||
it("reports frontmatter, directory, uniqueness, and inventory errors together", async () => {
|
||||
const packageDir = await createCatalogPackage();
|
||||
await writeSkill(packageDir, "bundled", "Bad_Category", "duplicate", {
|
||||
@@ -87,8 +163,8 @@ describe("skills catalog manifest", () => {
|
||||
|
||||
expect(result.errors).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.stringContaining("catalog/misc/SKILL.md is not under catalog/<bundled|optional>/<category>/<slug>/SKILL.md"),
|
||||
expect.stringContaining("catalog/bundled/software-development/missing-skill is missing SKILL.md"),
|
||||
expect.stringContaining("catalog/misc/SKILL.md is not under catalog/<bundled|optional>/<category>/<slug>/{SKILL.md,catalog-ref.json}"),
|
||||
expect.stringContaining("catalog/bundled/software-development/missing-skill is missing SKILL.md or catalog-ref.json"),
|
||||
expect.stringContaining("has invalid category"),
|
||||
expect.stringContaining("frontmatter must include description"),
|
||||
expect.stringContaining("key must be paperclipai/bundled/Bad_Category/duplicate"),
|
||||
@@ -163,3 +239,19 @@ async function writeSkill(
|
||||
await fs.writeFile(filePath, content, "utf8");
|
||||
}
|
||||
}
|
||||
|
||||
async function writeReference(
|
||||
packageDir: string,
|
||||
kind: "bundled" | "optional",
|
||||
category: string,
|
||||
slug: string,
|
||||
descriptor: Record<string, unknown>,
|
||||
) {
|
||||
const skillDir = path.join(packageDir, "catalog", kind, category, slug);
|
||||
await fs.mkdir(skillDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(skillDir, "catalog-ref.json"),
|
||||
`${JSON.stringify(descriptor, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
asBoolean,
|
||||
isPlainRecord,
|
||||
asString,
|
||||
asStringArray,
|
||||
parseFrontmatterMarkdown,
|
||||
@@ -14,23 +15,54 @@ import type {
|
||||
CatalogSkillFile,
|
||||
CatalogSkillFileKind,
|
||||
CatalogSkillKind,
|
||||
CatalogSkillSource,
|
||||
CatalogTrustLevel,
|
||||
} from "./types.js";
|
||||
|
||||
const CATALOG_PACKAGE_NAME = "@paperclipai/skills-catalog";
|
||||
const CATALOG_SCHEMA_VERSION = 1;
|
||||
const SKILL_ENTRYPOINT = "SKILL.md";
|
||||
const CATALOG_REFERENCE_FILE = "catalog-ref.json";
|
||||
const MAX_CATALOG_FILE_BYTES = 1024 * 1024;
|
||||
const SLUG_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
||||
const CATALOG_KINDS = new Set<CatalogSkillKind>(["bundled", "optional"]);
|
||||
|
||||
interface SkillCandidate {
|
||||
interface BaseSkillCandidate {
|
||||
kind: CatalogSkillKind;
|
||||
category: string;
|
||||
slug: string;
|
||||
absolutePath: string;
|
||||
}
|
||||
|
||||
type SkillCandidate =
|
||||
| (BaseSkillCandidate & { source: "local" })
|
||||
| (BaseSkillCandidate & { source: "reference"; descriptorPath: string });
|
||||
|
||||
interface ReferencedGitHubSourceDescriptor {
|
||||
type: "github";
|
||||
hostname?: string;
|
||||
owner: string;
|
||||
repo: string;
|
||||
ref: string;
|
||||
commit: string;
|
||||
path: string;
|
||||
}
|
||||
|
||||
interface ReferencedSkillDescriptor {
|
||||
source: ReferencedGitHubSourceDescriptor;
|
||||
files?: string[];
|
||||
defaultInstall?: boolean;
|
||||
recommendedForRoles?: string[];
|
||||
requires?: string[];
|
||||
tags?: string[];
|
||||
}
|
||||
|
||||
interface GitHubTreeEntry {
|
||||
path: string;
|
||||
type: "blob" | "tree" | string;
|
||||
size?: number;
|
||||
}
|
||||
|
||||
interface BuildCatalogManifestOptions {
|
||||
packageDir: string;
|
||||
generatedAt?: string;
|
||||
@@ -171,11 +203,23 @@ async function discoverSkillCandidates(packageDir: string, errors: string[]) {
|
||||
if (!slugEntry.isDirectory()) continue;
|
||||
const slug = slugEntry.name;
|
||||
const skillDir = path.join(categoryDir, slug);
|
||||
if (!existsSync(path.join(skillDir, SKILL_ENTRYPOINT))) {
|
||||
errors.push(`${relativePackagePath(packageDir, skillDir)} is missing SKILL.md.`);
|
||||
const hasLocalSkill = existsSync(path.join(skillDir, SKILL_ENTRYPOINT));
|
||||
const descriptorPath = path.join(skillDir, CATALOG_REFERENCE_FILE);
|
||||
const hasReference = existsSync(descriptorPath);
|
||||
|
||||
if (hasLocalSkill && hasReference) {
|
||||
errors.push(`${relativePackagePath(packageDir, skillDir)} must contain either SKILL.md or ${CATALOG_REFERENCE_FILE}, not both.`);
|
||||
continue;
|
||||
}
|
||||
candidates.push({ kind, category, slug, absolutePath: skillDir });
|
||||
if (!hasLocalSkill && !hasReference) {
|
||||
errors.push(`${relativePackagePath(packageDir, skillDir)} is missing SKILL.md or ${CATALOG_REFERENCE_FILE}.`);
|
||||
continue;
|
||||
}
|
||||
candidates.push(
|
||||
hasReference
|
||||
? { kind, category, slug, absolutePath: skillDir, source: "reference", descriptorPath }
|
||||
: { kind, category, slug, absolutePath: skillDir, source: "local" },
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -191,13 +235,13 @@ async function collectMisplacedSkillFiles(catalogDir: string, errors: string[])
|
||||
await visit(absolutePath);
|
||||
continue;
|
||||
}
|
||||
if (entry.name !== SKILL_ENTRYPOINT) continue;
|
||||
if (entry.name !== SKILL_ENTRYPOINT && entry.name !== CATALOG_REFERENCE_FILE) continue;
|
||||
|
||||
const relativePath = toPosixPath(path.relative(catalogDir, absolutePath));
|
||||
const parts = relativePath.split("/");
|
||||
const kind = parts[0];
|
||||
if (parts.length !== 4 || !CATALOG_KINDS.has(kind as CatalogSkillKind)) {
|
||||
errors.push(`catalog/${relativePath} is not under catalog/<bundled|optional>/<category>/<slug>/SKILL.md.`);
|
||||
errors.push(`catalog/${relativePath} is not under catalog/<bundled|optional>/<category>/<slug>/{SKILL.md,${CATALOG_REFERENCE_FILE}}.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -210,6 +254,10 @@ async function buildCatalogSkill(
|
||||
candidate: SkillCandidate,
|
||||
errors: string[],
|
||||
): Promise<CatalogSkill | null> {
|
||||
if (candidate.source === "reference") {
|
||||
return buildReferencedCatalogSkill(packageDir, candidate, errors);
|
||||
}
|
||||
|
||||
const prefix = relativePackagePath(packageDir, candidate.absolutePath);
|
||||
validateSlug("category", candidate.category, prefix, errors);
|
||||
validateSlug("slug", candidate.slug, prefix, errors);
|
||||
@@ -268,6 +316,268 @@ async function buildCatalogSkill(
|
||||
};
|
||||
}
|
||||
|
||||
async function buildReferencedCatalogSkill(
|
||||
packageDir: string,
|
||||
candidate: Extract<SkillCandidate, { source: "reference" }>,
|
||||
errors: string[],
|
||||
): Promise<CatalogSkill | null> {
|
||||
const prefix = relativePackagePath(packageDir, candidate.absolutePath);
|
||||
validateSlug("category", candidate.category, prefix, errors);
|
||||
validateSlug("slug", candidate.slug, prefix, errors);
|
||||
|
||||
const descriptor = await readReferencedSkillDescriptor(candidate.descriptorPath, prefix, errors);
|
||||
if (!descriptor) return null;
|
||||
|
||||
const id = `paperclipai:${candidate.kind}:${candidate.category}:${candidate.slug}`;
|
||||
const key = `paperclipai/${candidate.kind}/${candidate.category}/${candidate.slug}`;
|
||||
const source = buildCatalogSkillSource(descriptor.source, errors, `${prefix}/${CATALOG_REFERENCE_FILE}`);
|
||||
if (!source) return null;
|
||||
|
||||
const files = await collectReferencedSkillFiles(source, descriptor.files ?? [SKILL_ENTRYPOINT], prefix, errors);
|
||||
const skillMarkdown = await readReferencedFileText(source, SKILL_ENTRYPOINT, prefix, errors);
|
||||
if (!skillMarkdown) return null;
|
||||
|
||||
const parsed = parseFrontmatterMarkdown(skillMarkdown);
|
||||
if (!parsed.hasFrontmatter) {
|
||||
errors.push(`${source.url}/${SKILL_ENTRYPOINT} must start with YAML frontmatter.`);
|
||||
}
|
||||
|
||||
const name = asString(parsed.frontmatter.name);
|
||||
if (!name) errors.push(`${source.url}/${SKILL_ENTRYPOINT} frontmatter must include name.`);
|
||||
|
||||
const description = asString(parsed.frontmatter.description);
|
||||
if (!description) errors.push(`${source.url}/${SKILL_ENTRYPOINT} frontmatter must include description.`);
|
||||
|
||||
const explicitKey = asString(parsed.frontmatter.key);
|
||||
if (explicitKey && explicitKey !== key) {
|
||||
errors.push(`${source.url}/${SKILL_ENTRYPOINT} key must be ${key}.`);
|
||||
}
|
||||
|
||||
const explicitSlug = asString(parsed.frontmatter.slug);
|
||||
if (explicitSlug && explicitSlug !== candidate.slug) {
|
||||
errors.push(`${source.url}/${SKILL_ENTRYPOINT} slug must be ${candidate.slug}.`);
|
||||
}
|
||||
|
||||
const defaultInstall = asBoolean(descriptor.defaultInstall) ?? false;
|
||||
const recommendedForRoles = readStringArrayField(descriptor.recommendedForRoles, "recommendedForRoles", prefix, errors);
|
||||
const requires = readStringArrayField(descriptor.requires, "requires", prefix, errors);
|
||||
const tags = readStringArrayField(descriptor.tags, "tags", prefix, errors);
|
||||
|
||||
if (!files.some((file) => file.path === SKILL_ENTRYPOINT && file.kind === "skill")) {
|
||||
errors.push(`${prefix} referenced inventory does not contain SKILL.md.`);
|
||||
}
|
||||
if (!name || !description) return null;
|
||||
|
||||
return {
|
||||
id,
|
||||
key,
|
||||
kind: candidate.kind,
|
||||
category: candidate.category,
|
||||
slug: candidate.slug,
|
||||
name,
|
||||
description,
|
||||
path: toPosixPath(path.relative(packageDir, candidate.absolutePath)),
|
||||
entrypoint: SKILL_ENTRYPOINT,
|
||||
trustLevel: deriveTrustLevel(files),
|
||||
compatibility: "compatible",
|
||||
defaultInstall,
|
||||
recommendedForRoles,
|
||||
requires,
|
||||
tags,
|
||||
files,
|
||||
contentHash: buildContentHash(files),
|
||||
source,
|
||||
};
|
||||
}
|
||||
|
||||
async function readReferencedSkillDescriptor(
|
||||
descriptorPath: string,
|
||||
prefix: string,
|
||||
errors: string[],
|
||||
): Promise<ReferencedSkillDescriptor | null> {
|
||||
let raw: unknown;
|
||||
try {
|
||||
raw = JSON.parse(await fs.readFile(descriptorPath, "utf8"));
|
||||
} catch (error) {
|
||||
errors.push(`${prefix}/${CATALOG_REFERENCE_FILE} is missing or invalid JSON: ${errorMessage(error)}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!isPlainRecord(raw) || !isPlainRecord(raw.source)) {
|
||||
errors.push(`${prefix}/${CATALOG_REFERENCE_FILE} must include a source object.`);
|
||||
return null;
|
||||
}
|
||||
const sourceRaw = raw.source;
|
||||
if (asString(sourceRaw.type) !== "github") {
|
||||
errors.push(`${prefix}/${CATALOG_REFERENCE_FILE} source.type must be "github".`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const owner = asString(sourceRaw.owner);
|
||||
const repo = asString(sourceRaw.repo);
|
||||
const ref = asString(sourceRaw.ref);
|
||||
const commit = asString(sourceRaw.commit);
|
||||
const sourcePath = asString(sourceRaw.path);
|
||||
if (!owner || !repo || !ref || !commit || sourcePath === null) {
|
||||
errors.push(`${prefix}/${CATALOG_REFERENCE_FILE} GitHub source must include owner, repo, ref, commit, and path.`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const descriptor: ReferencedSkillDescriptor = {
|
||||
source: {
|
||||
type: "github",
|
||||
hostname: asString(sourceRaw.hostname) ?? "github.com",
|
||||
owner,
|
||||
repo,
|
||||
ref,
|
||||
commit,
|
||||
path: sourcePath,
|
||||
},
|
||||
defaultInstall: asBoolean(raw.defaultInstall) ?? false,
|
||||
files: asStringArray(raw.files ?? undefined) ?? undefined,
|
||||
recommendedForRoles: asStringArray(raw.recommendedForRoles ?? undefined) ?? undefined,
|
||||
requires: asStringArray(raw.requires ?? undefined) ?? undefined,
|
||||
tags: asStringArray(raw.tags ?? undefined) ?? undefined,
|
||||
};
|
||||
|
||||
if (raw.files !== undefined && !descriptor.files) errors.push(`${prefix}/${CATALOG_REFERENCE_FILE} files must be an array of strings.`);
|
||||
if (raw.recommendedForRoles !== undefined && !descriptor.recommendedForRoles) errors.push(`${prefix}/${CATALOG_REFERENCE_FILE} recommendedForRoles must be an array of strings.`);
|
||||
if (raw.requires !== undefined && !descriptor.requires) errors.push(`${prefix}/${CATALOG_REFERENCE_FILE} requires must be an array of strings.`);
|
||||
if (raw.tags !== undefined && !descriptor.tags) errors.push(`${prefix}/${CATALOG_REFERENCE_FILE} tags must be an array of strings.`);
|
||||
|
||||
return descriptor;
|
||||
}
|
||||
|
||||
function buildCatalogSkillSource(
|
||||
descriptor: ReferencedGitHubSourceDescriptor,
|
||||
errors: string[],
|
||||
prefix: string,
|
||||
): CatalogSkillSource | null {
|
||||
if (!/^[0-9a-f]{40}$/i.test(descriptor.commit)) {
|
||||
errors.push(`${prefix} source.commit must be a 40-character Git commit SHA.`);
|
||||
}
|
||||
const sourcePath = normalizeReferencedPath(descriptor.path);
|
||||
if (sourcePath === null) {
|
||||
errors.push(`${prefix} source.path must be a portable path within the repository.`);
|
||||
}
|
||||
const hostname = descriptor.hostname ?? "github.com";
|
||||
const url = `https://${hostname}/${descriptor.owner}/${descriptor.repo}/tree/${descriptor.ref}/${sourcePath ?? ""}`.replace(/\/$/, "");
|
||||
if (errors.length > 0 && (!/^[0-9a-f]{40}$/i.test(descriptor.commit) || sourcePath === null)) return null;
|
||||
return {
|
||||
type: "github",
|
||||
hostname,
|
||||
owner: descriptor.owner,
|
||||
repo: descriptor.repo,
|
||||
ref: descriptor.ref,
|
||||
commit: descriptor.commit,
|
||||
path: sourcePath ?? "",
|
||||
url,
|
||||
};
|
||||
}
|
||||
|
||||
async function collectReferencedSkillFiles(
|
||||
source: CatalogSkillSource,
|
||||
includePatterns: string[],
|
||||
prefix: string,
|
||||
errors: string[],
|
||||
): Promise<CatalogSkillFile[]> {
|
||||
const tree = await fetchGitHubTree(source, prefix, errors);
|
||||
const sourceRoot = source.path ? `${source.path}/` : "";
|
||||
const normalizedPatterns: string[] = [];
|
||||
for (const pattern of includePatterns) {
|
||||
const normalizedPattern = normalizeReferencedPath(pattern);
|
||||
if (normalizedPattern === null) {
|
||||
errors.push(`${prefix} referenced include path is invalid: ${pattern}`);
|
||||
continue;
|
||||
}
|
||||
if (normalizedPattern) normalizedPatterns.push(normalizedPattern);
|
||||
}
|
||||
const files: CatalogSkillFile[] = [];
|
||||
|
||||
for (const entry of tree) {
|
||||
if (entry.type !== "blob") continue;
|
||||
if (!entry.path.startsWith(sourceRoot)) continue;
|
||||
const relativePath = entry.path.slice(sourceRoot.length);
|
||||
if (!normalizedPatterns.some((pattern) => referencedPathMatches(relativePath, pattern))) continue;
|
||||
if ((entry.size ?? 0) > MAX_CATALOG_FILE_BYTES) {
|
||||
errors.push(`${prefix}/${relativePath} exceeds ${MAX_CATALOG_FILE_BYTES} bytes.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const bytes = await fetchReferencedFileBytes(source, relativePath, prefix, errors);
|
||||
if (!bytes) continue;
|
||||
files.push({
|
||||
path: relativePath,
|
||||
kind: classifyCatalogFile(relativePath),
|
||||
sizeBytes: bytes.byteLength,
|
||||
sha256: sha256(bytes),
|
||||
});
|
||||
}
|
||||
|
||||
files.sort((a, b) => {
|
||||
if (a.path === SKILL_ENTRYPOINT) return -1;
|
||||
if (b.path === SKILL_ENTRYPOINT) return 1;
|
||||
return a.path.localeCompare(b.path);
|
||||
});
|
||||
return files;
|
||||
}
|
||||
|
||||
async function fetchGitHubTree(
|
||||
source: CatalogSkillSource,
|
||||
prefix: string,
|
||||
errors: string[],
|
||||
): Promise<GitHubTreeEntry[]> {
|
||||
const url = `${githubApiBase(source.hostname)}/repos/${source.owner}/${source.repo}/git/trees/${source.commit}?recursive=1`;
|
||||
try {
|
||||
const response = await fetch(url, { headers: { accept: "application/vnd.github+json" } });
|
||||
if (!response.ok) {
|
||||
errors.push(`${prefix} failed to fetch GitHub tree: HTTP ${response.status}.`);
|
||||
return [];
|
||||
}
|
||||
const body = await response.json() as { tree?: GitHubTreeEntry[]; truncated?: boolean };
|
||||
if (body.truncated) errors.push(`${prefix} GitHub tree response was truncated.`);
|
||||
return Array.isArray(body.tree) ? body.tree : [];
|
||||
} catch (error) {
|
||||
errors.push(`${prefix} failed to fetch GitHub tree: ${errorMessage(error)}.`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function readReferencedFileText(
|
||||
source: CatalogSkillSource,
|
||||
relativePath: string,
|
||||
prefix: string,
|
||||
errors: string[],
|
||||
) {
|
||||
const bytes = await fetchReferencedFileBytes(source, relativePath, prefix, errors);
|
||||
return bytes ? bytes.toString("utf8") : null;
|
||||
}
|
||||
|
||||
async function fetchReferencedFileBytes(
|
||||
source: CatalogSkillSource,
|
||||
relativePath: string,
|
||||
prefix: string,
|
||||
errors: string[],
|
||||
): Promise<Buffer | null> {
|
||||
const normalizedPath = normalizeReferencedPath(relativePath);
|
||||
if (!normalizedPath) {
|
||||
errors.push(`${prefix} referenced file path is invalid: ${relativePath}`);
|
||||
return null;
|
||||
}
|
||||
const url = rawGitHubUrl(source, normalizedPath);
|
||||
try {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
errors.push(`${prefix}/${normalizedPath} failed to fetch pinned GitHub file: HTTP ${response.status}.`);
|
||||
return null;
|
||||
}
|
||||
return Buffer.from(await response.arrayBuffer());
|
||||
} catch (error) {
|
||||
errors.push(`${prefix}/${normalizedPath} failed to fetch pinned GitHub file: ${errorMessage(error)}.`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function collectSkillFiles(
|
||||
packageDir: string,
|
||||
skillDir: string,
|
||||
@@ -433,6 +743,40 @@ function toPosixPath(input: string) {
|
||||
return input.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function normalizeReferencedPath(input: string) {
|
||||
const normalized = input.replace(/\\/g, "/").replace(/^\/+/, "").replace(/\/+$/, "");
|
||||
if (normalized === "") return "";
|
||||
const parts = normalized.split("/");
|
||||
if (parts.includes("") || parts.includes(".") || parts.includes("..") || path.posix.isAbsolute(normalized)) {
|
||||
return null;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function referencedPathMatches(relativePath: string, pattern: string) {
|
||||
if (pattern.endsWith("/**")) {
|
||||
const directory = pattern.slice(0, -3);
|
||||
return relativePath === directory || relativePath.startsWith(`${directory}/`);
|
||||
}
|
||||
return relativePath === pattern;
|
||||
}
|
||||
|
||||
function githubApiBase(hostname: string) {
|
||||
const normalized = hostname.toLowerCase();
|
||||
return normalized === "github.com" || normalized === "www.github.com"
|
||||
? "https://api.github.com"
|
||||
: `https://${hostname}/api/v3`;
|
||||
}
|
||||
|
||||
function rawGitHubUrl(source: CatalogSkillSource, relativePath: string) {
|
||||
const fullPath = source.path ? `${source.path}/${relativePath}` : relativePath;
|
||||
const encodedPath = fullPath.split("/").map((segment) => encodeURIComponent(segment)).join("/");
|
||||
const normalized = source.hostname.toLowerCase();
|
||||
return normalized === "github.com" || normalized === "www.github.com"
|
||||
? `https://raw.githubusercontent.com/${source.owner}/${source.repo}/${source.commit}/${encodedPath}`
|
||||
: `https://${source.hostname}/raw/${source.owner}/${source.repo}/${source.commit}/${encodedPath}`;
|
||||
}
|
||||
|
||||
function isPathInside(parent: string, child: string) {
|
||||
const relativePath = path.relative(parent, child);
|
||||
return relativePath === "" || (!relativePath.startsWith("..") && !path.isAbsolute(relativePath));
|
||||
|
||||
@@ -7,7 +7,9 @@ export type {
|
||||
CatalogSkill,
|
||||
CatalogSkillFile,
|
||||
CatalogSkillFileKind,
|
||||
CatalogSkillGitHubSource,
|
||||
CatalogSkillKind,
|
||||
CatalogSkillSource,
|
||||
CatalogTrustLevel,
|
||||
CatalogValidationResult,
|
||||
} from "./types.js";
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { catalogManifest, catalogSkills, resolveCatalogSkillRef } from "./index.js";
|
||||
import type { CatalogSkill } from "./types.js";
|
||||
|
||||
const EXPECTED_BUNDLED_KEYS = [
|
||||
"paperclipai/bundled/docs/doc-maintenance",
|
||||
@@ -15,6 +14,7 @@ const EXPECTED_OPTIONAL_KEYS = [
|
||||
"paperclipai/optional/browser/agent-browser",
|
||||
"paperclipai/optional/content/release-announcement",
|
||||
"paperclipai/optional/product/design-critique",
|
||||
"paperclipai/optional/research/last30days",
|
||||
];
|
||||
|
||||
describe("shipped skills catalog", () => {
|
||||
@@ -32,12 +32,14 @@ describe("shipped skills catalog", () => {
|
||||
expect(optionalKeys).toEqual(EXPECTED_OPTIONAL_KEYS);
|
||||
});
|
||||
|
||||
it("keeps every shipped skill free of executable scripts until script-bearing skills clear security review", () => {
|
||||
// The real install-time security boundary (server assertCatalogSkillInstallable) blocks
|
||||
// only "scripts_executables". Static assets (svg/html templates, e.g. the wireframe skill)
|
||||
// carry the "assets" trust level and are installable, so they are allowed in the catalog.
|
||||
it("keeps script-bearing shipped skills explicit so install stays audit-gated", () => {
|
||||
// The real install-time security boundary audits materialized bytes and blocks
|
||||
// hard-stop findings. Static assets (svg/html templates, e.g. the wireframe skill)
|
||||
// carry the "assets" trust level and are installable.
|
||||
const scriptBearing = catalogSkills.filter((skill) => skill.trustLevel === "scripts_executables");
|
||||
expect(scriptBearing, formatViolations("script-bearing skills require security review", scriptBearing)).toEqual([]);
|
||||
expect(scriptBearing.map((skill) => skill.key)).toEqual([
|
||||
"paperclipai/optional/research/last30days",
|
||||
]);
|
||||
});
|
||||
|
||||
it("populates browse/search-relevant fields for every shipped skill", () => {
|
||||
@@ -86,9 +88,3 @@ describe("shipped skills catalog", () => {
|
||||
expect(resolveCatalogSkillRef(sample.slug)).toMatchObject({ key: sample.key });
|
||||
});
|
||||
});
|
||||
|
||||
function formatViolations(label: string, skills: CatalogSkill[]) {
|
||||
if (skills.length === 0) return label;
|
||||
const detail = skills.map((skill) => `${skill.key} (${skill.trustLevel})`).join(", ");
|
||||
return `${label}: ${detail}`;
|
||||
}
|
||||
|
||||
@@ -13,6 +13,19 @@ export interface CatalogSkillFile {
|
||||
sha256: string;
|
||||
}
|
||||
|
||||
export interface CatalogSkillGitHubSource {
|
||||
type: "github";
|
||||
hostname: string;
|
||||
owner: string;
|
||||
repo: string;
|
||||
ref: string;
|
||||
commit: string;
|
||||
path: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export type CatalogSkillSource = CatalogSkillGitHubSource;
|
||||
|
||||
export interface CatalogSkill {
|
||||
id: string;
|
||||
key: string;
|
||||
@@ -31,6 +44,7 @@ export interface CatalogSkill {
|
||||
tags: string[];
|
||||
files: CatalogSkillFile[];
|
||||
contentHash: string;
|
||||
source?: CatalogSkillSource;
|
||||
}
|
||||
|
||||
export interface CatalogManifest {
|
||||
|
||||
Reference in New Issue
Block a user