diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 1c552a1f..f6861f8d 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -315,6 +315,8 @@ export type { CatalogSkillKind, CatalogSkillFileKind, CatalogSkillFile, + CatalogSkillGitHubSource, + CatalogSkillSource, CatalogSkill, CatalogSkillListQuery, CatalogSkillFileDetail, @@ -1099,6 +1101,8 @@ export { companySkillFileUpdateSchema, catalogSkillKindSchema, catalogSkillFileSchema, + catalogSkillGitHubSourceSchema, + catalogSkillSourceSchema, catalogSkillSchema, catalogSkillListQuerySchema, catalogSkillFileDetailSchema, diff --git a/packages/shared/src/types/company-skill.ts b/packages/shared/src/types/company-skill.ts index b244b71c..356e3d42 100644 --- a/packages/shared/src/types/company-skill.ts +++ b/packages/shared/src/types/company-skill.ts @@ -214,6 +214,19 @@ export interface CatalogSkillFile { sha256: string; } +export interface CatalogSkillGitHubSource { + type: "github"; + hostname: string; + owner: string; + repo: string; + ref: string; + commit: string; + path: string; + url: string; +} + +export type CatalogSkillSource = CatalogSkillGitHubSource; + export interface CatalogSkill { id: string; key: string; @@ -232,6 +245,7 @@ export interface CatalogSkill { tags: string[]; files: CatalogSkillFile[]; contentHash: string; + source?: CatalogSkillSource; packageName?: string; packageVersion?: string; } diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index e3c80954..7a8146c3 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -70,6 +70,8 @@ export type { CatalogSkillKind, CatalogSkillFileKind, CatalogSkillFile, + CatalogSkillGitHubSource, + CatalogSkillSource, CatalogSkill, CatalogSkillListQuery, CatalogSkillFileDetail, diff --git a/packages/shared/src/validators/company-skill.test.ts b/packages/shared/src/validators/company-skill.test.ts index c9524238..20965e63 100644 --- a/packages/shared/src/validators/company-skill.test.ts +++ b/packages/shared/src/validators/company-skill.test.ts @@ -28,6 +28,16 @@ const catalogSkill = { tags: ["review"], files: [{ path: "SKILL.md", kind: "skill", sizeBytes: 8, sha256: "abc" }], contentHash: "sha256:abc", + source: { + type: "github", + hostname: "github.com", + owner: "example", + repo: "review-skill", + ref: "v1.0.0", + commit: "0123456789abcdef0123456789abcdef01234567", + path: "skills/review", + url: "https://github.com/example/review-skill/tree/v1.0.0/skills/review", + }, }; const companySkill = { @@ -107,6 +117,7 @@ describe("company skill catalog validators", () => { }, catalogSkill: { id: catalogSkill.id, + source: catalogSkill.source, }, }); }); diff --git a/packages/shared/src/validators/company-skill.ts b/packages/shared/src/validators/company-skill.ts index 2d813ec0..a0bbb732 100644 --- a/packages/shared/src/validators/company-skill.ts +++ b/packages/shared/src/validators/company-skill.ts @@ -182,6 +182,19 @@ export const catalogSkillFileSchema = z.object({ sha256: z.string().min(1), }); +export const catalogSkillGitHubSourceSchema = z.object({ + type: z.literal("github"), + hostname: z.string().min(1), + owner: z.string().min(1), + repo: z.string().min(1), + ref: z.string().min(1), + commit: z.string().regex(/^[0-9a-f]{40}$/i), + path: z.string(), + url: z.string().url(), +}); + +export const catalogSkillSourceSchema = catalogSkillGitHubSourceSchema; + export const catalogSkillSchema = z.object({ id: z.string().min(1), key: z.string().min(1), @@ -200,6 +213,7 @@ export const catalogSkillSchema = z.object({ tags: z.array(z.string()), files: z.array(catalogSkillFileSchema), contentHash: z.string().min(1), + source: catalogSkillSourceSchema.optional(), packageName: z.string().min(1).optional(), packageVersion: z.string().min(1).optional(), }); diff --git a/packages/shared/src/validators/index.ts b/packages/shared/src/validators/index.ts index 79b80161..083f3fe3 100644 --- a/packages/shared/src/validators/index.ts +++ b/packages/shared/src/validators/index.ts @@ -79,6 +79,8 @@ export { companySkillFileUpdateSchema, catalogSkillKindSchema, catalogSkillFileSchema, + catalogSkillGitHubSourceSchema, + catalogSkillSourceSchema, catalogSkillSchema, catalogSkillListQuerySchema, catalogSkillFileDetailSchema, diff --git a/packages/skills-catalog/catalog/optional/research/last30days/catalog-ref.json b/packages/skills-catalog/catalog/optional/research/last30days/catalog-ref.json new file mode 100644 index 00000000..314a149c --- /dev/null +++ b/packages/skills-catalog/catalog/optional/research/last30days/catalog-ref.json @@ -0,0 +1,44 @@ +{ + "source": { + "type": "github", + "hostname": "github.com", + "owner": "mvanhorn", + "repo": "last30days-skill", + "ref": "v3.3.0", + "commit": "daca71f89eb71d0d56d01a43ed7627aa919dba4f", + "path": "skills/last30days" + }, + "files": [ + "SKILL.md", + "agents/openai.yaml", + "references/**", + "scripts/briefing.py", + "scripts/compare.sh", + "scripts/last30days.py", + "scripts/lib/**", + "scripts/setup-keychain.sh", + "scripts/store.py", + "scripts/watchlist.py" + ], + "defaultInstall": false, + "recommendedForRoles": [ + "researcher", + "marketer", + "product-manager", + "analyst" + ], + "requires": [ + "node", + "python3" + ], + "tags": [ + "research", + "last-30-days", + "social-media", + "trends", + "citations", + "reddit", + "x", + "youtube" + ] +} diff --git a/packages/skills-catalog/generated/catalog.json b/packages/skills-catalog/generated/catalog.json index cdcb7b38..be85c448 100644 --- a/packages/skills-catalog/generated/catalog.json +++ b/packages/skills-catalog/generated/catalog.json @@ -2,7 +2,7 @@ "schemaVersion": 1, "packageName": "@paperclipai/skills-catalog", "packageVersion": "0.3.1", - "generatedAt": "2026-05-31T18:06:11.126Z", + "generatedAt": "2026-06-04T19:57:14.020Z", "skills": [ { "id": "paperclipai:bundled:docs:doc-maintenance", @@ -352,6 +352,527 @@ } ], "contentHash": "sha256:429f94df398a0697042b5bbe4755b1ff1a230aa5f41d99118ad37493ac65d21c" + }, + { + "id": "paperclipai:optional:research:last30days", + "key": "paperclipai/optional/research/last30days", + "kind": "optional", + "category": "research", + "slug": "last30days", + "name": "last30days", + "description": "Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web.", + "path": "catalog/optional/research/last30days", + "entrypoint": "SKILL.md", + "trustLevel": "scripts_executables", + "compatibility": "compatible", + "defaultInstall": false, + "recommendedForRoles": [ + "researcher", + "marketer", + "product-manager", + "analyst" + ], + "requires": [ + "node", + "python3" + ], + "tags": [ + "research", + "last-30-days", + "social-media", + "trends", + "citations", + "reddit", + "x", + "youtube" + ], + "files": [ + { + "path": "SKILL.md", + "kind": "skill", + "sizeBytes": 130813, + "sha256": "534f0347f304ec51254a7d3a210e07c86cc252175cc6a588e97d32ce94f0f745" + }, + { + "path": "agents/openai.yaml", + "kind": "other", + "sizeBytes": 448, + "sha256": "f46e7f09d3118dee7324ac165391ccb7ffe55323d1d75304e34062017913939c" + }, + { + "path": "references/save-html-brief.md", + "kind": "reference", + "sizeBytes": 5940, + "sha256": "ca15e03bf63a3af1cfa0347df36b0aee999c6c6c631f31b4171a5df09e765d43" + }, + { + "path": "scripts/briefing.py", + "kind": "script", + "sizeBytes": 8342, + "sha256": "d4881d16493f5504c5a947f31ced861d674e2423e5fc668305beaf5e849420b7" + }, + { + "path": "scripts/compare.sh", + "kind": "script", + "sizeBytes": 2291, + "sha256": "cdb2538522a60c803aeebc3336673c8702380a902604ad85770206382996a6f5" + }, + { + "path": "scripts/last30days.py", + "kind": "script", + "sizeBytes": 44282, + "sha256": "818d6b839eb04417a5d0acf3bee24258fbc70543df968619fd3573597dc88e8e" + }, + { + "path": "scripts/lib/__init__.py", + "kind": "script", + "sizeBytes": 29, + "sha256": "3fb925405f898201188f9b5eea04e214ea8c8a64643397fd727f08ffeea234d9" + }, + { + "path": "scripts/lib/bird_x.py", + "kind": "script", + "sizeBytes": 17960, + "sha256": "c7cbd33a5b4c317b07eeca7c7e3c1aa57369487cd4cd3602abbe9b6acb019e6a" + }, + { + "path": "scripts/lib/bluesky.py", + "kind": "script", + "sizeBytes": 12343, + "sha256": "af7bbaa68f6944a50089447992b95881e15c085de829426360f50740a35cba89" + }, + { + "path": "scripts/lib/categories.py", + "kind": "script", + "sizeBytes": 7635, + "sha256": "e4ab8435e09c371f2475751fdc1c58150799b3746afcc7e38d5dad30adb210d5" + }, + { + "path": "scripts/lib/chrome_cookies.py", + "kind": "script", + "sizeBytes": 10911, + "sha256": "eeb8f99b92cfc9aa65b20b6dbedfcfd321166f3fa771a95299ffb0b950b1cedc" + }, + { + "path": "scripts/lib/cluster.py", + "kind": "script", + "sizeBytes": 11018, + "sha256": "fa9fb8905552e7bb5a2a2854f8b3c620a464054238a008d74ba89d0c247a776e" + }, + { + "path": "scripts/lib/competitors.py", + "kind": "script", + "sizeBytes": 7332, + "sha256": "a76c5d679b7f3c93a07e389811f79526f11e1e03411cecd4d9569e3aca32a77d" + }, + { + "path": "scripts/lib/cookie_extract.py", + "kind": "script", + "sizeBytes": 14450, + "sha256": "9d34ea3a8565eae8b67f72e8b625c22a29d41274c3b8d6c8b6aa171c18ebad5b" + }, + { + "path": "scripts/lib/dates.py", + "kind": "script", + "sizeBytes": 3201, + "sha256": "ae4ec6455021c82ef3321ef5dde6f7ba41e6b91a3e6ff601174ac9aa20e5c8f9" + }, + { + "path": "scripts/lib/dedupe.py", + "kind": "script", + "sizeBytes": 3315, + "sha256": "b9acabf496e0743189fbeed43d447280fee17e10a40931f9083d2a1f2894d863" + }, + { + "path": "scripts/lib/digg.py", + "kind": "script", + "sizeBytes": 13305, + "sha256": "95a2845d4751efb5738d2a69c7246f8269447395a7da324e0d6ae0feefd7c9d8" + }, + { + "path": "scripts/lib/entity_extract.py", + "kind": "script", + "sizeBytes": 4211, + "sha256": "b04103103981915557fd410305f3a8031d5e71d687491ca89976be25d9bc84a4" + }, + { + "path": "scripts/lib/env.py", + "kind": "script", + "sizeBytes": 25940, + "sha256": "9ca2e2341565a6c933d51800a510d07b84707825d19234d991eb472454b46933" + }, + { + "path": "scripts/lib/fanout.py", + "kind": "script", + "sizeBytes": 3155, + "sha256": "43f21244f9bd1bf143d2f4c478cc2fcc9ffc4e403539e8c08e86e106ed5f0b83" + }, + { + "path": "scripts/lib/fusion.py", + "kind": "script", + "sizeBytes": 8553, + "sha256": "c764709a7d4576121c193ae75ded057dd177b6bea512ca57dd72680d74030b41" + }, + { + "path": "scripts/lib/github.py", + "kind": "script", + "sizeBytes": 34879, + "sha256": "0f8eb97c20463b975b3bb752d0e3afec98ef853430e495f21fc06f01c27d2baf" + }, + { + "path": "scripts/lib/grounding.py", + "kind": "script", + "sizeBytes": 11723, + "sha256": "440386c32afb3c01678fe5dfab197449a9a910018a40cbbc1e9e10ecb9d4843a" + }, + { + "path": "scripts/lib/hackernews.py", + "kind": "script", + "sizeBytes": 11619, + "sha256": "95db30a3bf441ee73fb49a04fa8974440e381e56c7d70e87b48e3b69ae0cdfa5" + }, + { + "path": "scripts/lib/html_render.py", + "kind": "script", + "sizeBytes": 17791, + "sha256": "5070f6a58d6fcae44a20ecd9108fd204b042bea1d47f8a67efce6d3831aa87ad" + }, + { + "path": "scripts/lib/http.py", + "kind": "script", + "sizeBytes": 10191, + "sha256": "760308aa8b8e3abbaa3273a4ca4df5b0bc8cc71ba8864b5740aaac545972731e" + }, + { + "path": "scripts/lib/instagram.py", + "kind": "script", + "sizeBytes": 18874, + "sha256": "cda8c8558e6cdc9b4bf3f7bb470c2e223db927acc6a77f139a83ec7987488bb1" + }, + { + "path": "scripts/lib/log.py", + "kind": "script", + "sizeBytes": 831, + "sha256": "11c4d61cdb5b1ff91919dcd28ec62ce88e555def30eed62a8071973b5ffe8317" + }, + { + "path": "scripts/lib/normalize.py", + "kind": "script", + "sizeBytes": 19870, + "sha256": "e79e294e4ff8ebfa34f7561c63e0846af0380b8be299d484cc8a2f490e82b474" + }, + { + "path": "scripts/lib/perplexity.py", + "kind": "script", + "sizeBytes": 4789, + "sha256": "70859e7296035936ae0da441119edfe381158d5d0de06afee87a3504007f91c7" + }, + { + "path": "scripts/lib/pinterest.py", + "kind": "script", + "sizeBytes": 5366, + "sha256": "a6ac5085c900072b84930fafab8c81c9e5a792a718c21801d3fa0bd293d279f6" + }, + { + "path": "scripts/lib/pipeline.py", + "kind": "script", + "sizeBytes": 45491, + "sha256": "9641f56faff2d685b0c5c483b2113721d151174a2eef265afe644e131c554454" + }, + { + "path": "scripts/lib/planner.py", + "kind": "script", + "sizeBytes": 30490, + "sha256": "c0953abea402454a88c2573aa0c97bb6bba4c87e529d9f157a881e15c4123154" + }, + { + "path": "scripts/lib/polymarket.py", + "kind": "script", + "sizeBytes": 30318, + "sha256": "cbd54a66988b4ebc4aebf2db6c637d4e516065518c333cd0495332458c2d9080" + }, + { + "path": "scripts/lib/preflight.py", + "kind": "script", + "sizeBytes": 4541, + "sha256": "0fafa53bc63a44ad4525d99d6bb37080b0693e2d45265ce36af09b4a0e9b2c9b" + }, + { + "path": "scripts/lib/providers.py", + "kind": "script", + "sizeBytes": 15817, + "sha256": "7460004a2e8df0f8c94d764c0eb01f8d782bb160ce2e157105fadd0dd2953a5d" + }, + { + "path": "scripts/lib/quality_nudge.py", + "kind": "script", + "sizeBytes": 13438, + "sha256": "69b6d58abea8171c895f105ee62b2db6f7ea35cebbe08d7e6e8884d810891fef" + }, + { + "path": "scripts/lib/query.py", + "kind": "script", + "sizeBytes": 4018, + "sha256": "6bc2e0649034cdb28471346a66e2448ba58762cb7774c4fce45c055ab160912a" + }, + { + "path": "scripts/lib/reddit_enrich.py", + "kind": "script", + "sizeBytes": 9535, + "sha256": "652a9c5abecf22f3c26ff6d6b732a43703113a559ce31acab7af012eb25d4e7a" + }, + { + "path": "scripts/lib/reddit_public.py", + "kind": "script", + "sizeBytes": 13066, + "sha256": "ea25034726aa895c123216e356c03856a8116efedd29620bb2ebc0d01ff4c420" + }, + { + "path": "scripts/lib/reddit.py", + "kind": "script", + "sizeBytes": 24701, + "sha256": "dba1019f67f677c869f14a1f892b1562dd10b6f96bffc75ffad37eb4d3bc9877" + }, + { + "path": "scripts/lib/relevance.py", + "kind": "script", + "sizeBytes": 5941, + "sha256": "b9f776f9f8e5d4fbf1fcc86f6604bc0c391c988153d3264b8dc431ff34783382" + }, + { + "path": "scripts/lib/render.py", + "kind": "script", + "sizeBytes": 71577, + "sha256": "189ffa786428515a3b02f680e71069b7907eaa24c7aadd315e9abaecb0a6a9fc" + }, + { + "path": "scripts/lib/rerank.py", + "kind": "script", + "sizeBytes": 17106, + "sha256": "419c52fc08144c41e08f7574093c5b6cd21271900b51680b6393e61441ffea3c" + }, + { + "path": "scripts/lib/resolve.py", + "kind": "script", + "sizeBytes": 12448, + "sha256": "b23f3dd54d6ede0986219aacdda81ece72f1a0a629d72a9ec56b24bc645ccb7c" + }, + { + "path": "scripts/lib/safari_cookies.py", + "kind": "script", + "sizeBytes": 5698, + "sha256": "4d66c27cbf5725cc7cb815ca42191e70261c52196c2823bab9c6050ca32ccf71" + }, + { + "path": "scripts/lib/schema.py", + "kind": "script", + "sizeBytes": 11206, + "sha256": "8d815d3daee8d469e504bb1264fc174af4abc179dd7d1fa87d7313ed01d914c6" + }, + { + "path": "scripts/lib/setup_wizard.py", + "kind": "script", + "sizeBytes": 18688, + "sha256": "be0f867e96d15f7458faac2194c444dc0ec9ec4f3c9fe4d74459e083b80c1a73" + }, + { + "path": "scripts/lib/signals.py", + "kind": "script", + "sizeBytes": 9715, + "sha256": "4071cf5fa4c99b0ee51d949ca69ec4594d7e1986ec6a8019b55bb877e219d49a" + }, + { + "path": "scripts/lib/skill_meta.py", + "kind": "script", + "sizeBytes": 1207, + "sha256": "e922e6ecdb19fa3daf9c6673acb13a9b23f97362eac1cb11211c998dea0fdbd1" + }, + { + "path": "scripts/lib/snippet.py", + "kind": "script", + "sizeBytes": 1560, + "sha256": "6bd2143a4264fe11d5be1b805742336ab82397ae0293c13e8b5b3084bfa2612d" + }, + { + "path": "scripts/lib/subproc.py", + "kind": "script", + "sizeBytes": 2807, + "sha256": "cf7bf3dcaf79375c5bd014718a8eb5f6565275294eeaa0059fef93d8e4b6452d" + }, + { + "path": "scripts/lib/threads.py", + "kind": "script", + "sizeBytes": 6728, + "sha256": "7508629028c3b4a187699797ab0d9e7b179cbae3c53b9fabdaf9d905b1fc2c1d" + }, + { + "path": "scripts/lib/tiktok.py", + "kind": "script", + "sizeBytes": 20993, + "sha256": "f0074737c4bbf176118382ad4c0c9e8c8d6f4544bd9fc42d224f7c1da13a8d73" + }, + { + "path": "scripts/lib/truthsocial.py", + "kind": "script", + "sizeBytes": 5196, + "sha256": "fd745d0a61b1c69bea813832f3e2c5923b08d476766a378c07f0963b0be2f330" + }, + { + "path": "scripts/lib/ui.py", + "kind": "script", + "sizeBytes": 25907, + "sha256": "cdc0ea376fc4ee017449f2aa87a7513add3fa5306f27d4663fb6f7203f12974e" + }, + { + "path": "scripts/lib/vendor/bird-search/bird-search.mjs", + "kind": "script", + "sizeBytes": 3998, + "sha256": "584482e894c38bbde7a915013f741974fa8ea722ac34c56704dccf184f7c3d56" + }, + { + "path": "scripts/lib/vendor/bird-search/lib/cookies.js", + "kind": "script", + "sizeBytes": 8153, + "sha256": "66044440cef3d3b3e3136091b6e224051e8ff97fb1d156f40ccf2d4f3cf203ae" + }, + { + "path": "scripts/lib/vendor/bird-search/lib/features.json", + "kind": "script", + "sizeBytes": 523, + "sha256": "36361f4cae0040255f84709459ebb1f0131566a567269ed7b99d073cad930f66" + }, + { + "path": "scripts/lib/vendor/bird-search/lib/paginate-cursor.js", + "kind": "script", + "sizeBytes": 1225, + "sha256": "5f20227de1d68250ad59b7ad593db4bfb1d0f1a21694963eab3ed413c2d0d5c0" + }, + { + "path": "scripts/lib/vendor/bird-search/lib/query-ids.json", + "kind": "script", + "sizeBytes": 815, + "sha256": "eba47a9d26f87d0b4eb8a4f554833ed8d11b3d8728e363deeae702d09f87fc02" + }, + { + "path": "scripts/lib/vendor/bird-search/lib/runtime-features.js", + "kind": "script", + "sizeBytes": 5036, + "sha256": "f41614d62b6c91dfc85fc4af389e6f49435324bf26460b715a991486609579d8" + }, + { + "path": "scripts/lib/vendor/bird-search/lib/runtime-query-ids.js", + "kind": "script", + "sizeBytes": 9424, + "sha256": "0f5c61dbaad3de7fb44a14c57ed005228e51f9fc361472eff59bff4cb2a1199e" + }, + { + "path": "scripts/lib/vendor/bird-search/lib/twitter-client-base.js", + "kind": "script", + "sizeBytes": 4853, + "sha256": "ea972b7894458e78edd1927d102a052ae45726a7c967b376ac7f5544f5b65499" + }, + { + "path": "scripts/lib/vendor/bird-search/lib/twitter-client-constants.js", + "kind": "script", + "sizeBytes": 2535, + "sha256": "27bd1416ff1f5d4f4516d784a8fb9ee08a87413f07b375752821a32e5b0cd6e3" + }, + { + "path": "scripts/lib/vendor/bird-search/lib/twitter-client-features.js", + "kind": "script", + "sizeBytes": 18400, + "sha256": "4b4f2048069f751a821777cb2084f3b9126d99a958494fef71f0699d8145c4dd" + }, + { + "path": "scripts/lib/vendor/bird-search/lib/twitter-client-search.js", + "kind": "script", + "sizeBytes": 7310, + "sha256": "3dfbe18ea54215d9a6393ff0884d9977dd9ddf0884a06486bdaee8a4e30d4616" + }, + { + "path": "scripts/lib/vendor/bird-search/lib/twitter-client-types.js", + "kind": "script", + "sizeBytes": 59, + "sha256": "6450f74c664db958d703e09288184ef1ca5f622782f9e11a5a9bfa3d3c78c534" + }, + { + "path": "scripts/lib/vendor/bird-search/lib/twitter-client-utils.js", + "kind": "script", + "sizeBytes": 19169, + "sha256": "b230db5583760ccd0b5b5fb1592f449a7d0bc92f6184808e8ffb6f8968df20e9" + }, + { + "path": "scripts/lib/vendor/bird-search/LICENSE", + "kind": "script", + "sizeBytes": 1074, + "sha256": "62316704df7426e5a79d2827ff8aca36e9abb3a73b8e68557030749ebefec667" + }, + { + "path": "scripts/lib/vendor/bird-search/package.json", + "kind": "script", + "sizeBytes": 331, + "sha256": "96b65305969a179ee7a7735f1b475599d6035f32fda1fd030ea57ee09380cdb5" + }, + { + "path": "scripts/lib/xai_x.py", + "kind": "script", + "sizeBytes": 7509, + "sha256": "6cc244293ed6ea3b0f78a6515b583ff36da96ad30acf1d5457278ab020f1bd47" + }, + { + "path": "scripts/lib/xiaohongshu_api.py", + "kind": "script", + "sizeBytes": 5321, + "sha256": "c7a29dfa03f871993dc13a8ad528beed8a648102404b658242ca57878077ef60" + }, + { + "path": "scripts/lib/xquik.py", + "kind": "script", + "sizeBytes": 6839, + "sha256": "7c487cb2ba543ff12e38ec6466a2de6ff6762ccd415c1f07093620b49137681a" + }, + { + "path": "scripts/lib/xurl_x.py", + "kind": "script", + "sizeBytes": 5302, + "sha256": "8cf6bb235cfc7f39386b282a922487816089fa6ebf4fa49a4d562d95f9686e78" + }, + { + "path": "scripts/lib/youtube_yt.py", + "kind": "script", + "sizeBytes": 37339, + "sha256": "787908d06a845ad70654c4939ebabc856311a95b34444f0987ff7ee3db4c0a49" + }, + { + "path": "scripts/setup-keychain.sh", + "kind": "script", + "sizeBytes": 3229, + "sha256": "5d2a6b678c905eefe537fbcb98063db3e471c22b7ab4b06d51615d99b14556db" + }, + { + "path": "scripts/store.py", + "kind": "script", + "sizeBytes": 28534, + "sha256": "c1e95a00132768e6c38170bee85161a59b5bce3feaa6297a85f1a49d1bfc9fd3" + }, + { + "path": "scripts/watchlist.py", + "kind": "script", + "sizeBytes": 8929, + "sha256": "c06378f70f9145f3bd957170dec2549a31463a41b400bbf1ce722911aa2150a1" + } + ], + "contentHash": "sha256:9e11e9d2ed3d2062273d131a826cdeedea39bdd269063e2f73ca7f218770c769", + "source": { + "type": "github", + "hostname": "github.com", + "owner": "mvanhorn", + "repo": "last30days-skill", + "ref": "v3.3.0", + "commit": "daca71f89eb71d0d56d01a43ed7627aa919dba4f", + "path": "skills/last30days", + "url": "https://github.com/mvanhorn/last30days-skill/tree/v3.3.0/skills/last30days" + } } ] } diff --git a/packages/skills-catalog/src/catalog-builder.test.ts b/packages/skills-catalog/src/catalog-builder.test.ts index 9f06a761..8f0f1bd2 100644 --- a/packages/skills-catalog/src/catalog-builder.test.ts +++ b/packages/skills-catalog/src/catalog-builder.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { buildCatalogManifest, formatCatalogManifest, @@ -13,6 +13,7 @@ const tempDirs: string[] = []; describe("skills catalog manifest", () => { afterEach(async () => { await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); + vi.unstubAllGlobals(); }); it("builds stable manifest entries from catalog skill directories", async () => { @@ -59,6 +60,81 @@ describe("skills catalog manifest", () => { expect(result.manifest.skills[0]!.contentHash).toMatch(/^sha256:[a-f0-9]{64}$/); }); + it("builds stable manifest entries from pinned GitHub references", async () => { + const packageDir = await createCatalogPackage(); + const skillMarkdown = [ + "---", + "name: Remote Research", + "description: Research recent discussion from a pinned upstream skill.", + "---", + "", + "Use this skill.", + "", + ].join("\n"); + const script = "print('hello')\n"; + await writeReference(packageDir, "optional", "research", "remote-research", { + source: { + type: "github", + hostname: "github.com", + owner: "example", + repo: "remote-skill", + ref: "v1.0.0", + commit: "0123456789abcdef0123456789abcdef01234567", + path: "skills/remote-research", + }, + files: ["SKILL.md", "scripts/**"], + recommendedForRoles: ["researcher"], + tags: ["research"], + }); + const fetchMock = vi.fn(async (url: string) => { + if (url.includes("/git/trees/")) { + return new Response(JSON.stringify({ + tree: [ + { path: "skills/remote-research/SKILL.md", type: "blob", size: Buffer.byteLength(skillMarkdown) }, + { path: "skills/remote-research/scripts/run.py", type: "blob", size: Buffer.byteLength(script) }, + { path: "README.md", type: "blob", size: 9 }, + ], + }), { status: 200 }); + } + if (url.endsWith("/skills/remote-research/SKILL.md")) { + return new Response(skillMarkdown, { status: 200 }); + } + if (url.endsWith("/skills/remote-research/scripts/run.py")) { + return new Response(script, { status: 200 }); + } + return new Response("not found", { status: 404 }); + }); + vi.stubGlobal("fetch", fetchMock); + + const result = await buildCatalogManifest({ + packageDir, + generatedAt: "2026-05-26T00:00:00.000Z", + }); + + expect(result.errors).toEqual([]); + expect(result.manifest.skills[0]).toMatchObject({ + id: "paperclipai:optional:research:remote-research", + key: "paperclipai/optional/research/remote-research", + path: "catalog/optional/research/remote-research", + trustLevel: "scripts_executables", + recommendedForRoles: ["researcher"], + tags: ["research"], + source: { + type: "github", + owner: "example", + repo: "remote-skill", + ref: "v1.0.0", + commit: "0123456789abcdef0123456789abcdef01234567", + path: "skills/remote-research", + }, + }); + expect(result.manifest.skills[0]!.files.map((file) => file.path)).toEqual([ + "SKILL.md", + "scripts/run.py", + ]); + expect(result.manifest.skills[0]!.contentHash).toMatch(/^sha256:[a-f0-9]{64}$/); + }); + it("reports frontmatter, directory, uniqueness, and inventory errors together", async () => { const packageDir = await createCatalogPackage(); await writeSkill(packageDir, "bundled", "Bad_Category", "duplicate", { @@ -87,8 +163,8 @@ describe("skills catalog manifest", () => { expect(result.errors).toEqual( expect.arrayContaining([ - expect.stringContaining("catalog/misc/SKILL.md is not under catalog////SKILL.md"), - expect.stringContaining("catalog/bundled/software-development/missing-skill is missing SKILL.md"), + expect.stringContaining("catalog/misc/SKILL.md is not under catalog////{SKILL.md,catalog-ref.json}"), + expect.stringContaining("catalog/bundled/software-development/missing-skill is missing SKILL.md or catalog-ref.json"), expect.stringContaining("has invalid category"), expect.stringContaining("frontmatter must include description"), expect.stringContaining("key must be paperclipai/bundled/Bad_Category/duplicate"), @@ -163,3 +239,19 @@ async function writeSkill( await fs.writeFile(filePath, content, "utf8"); } } + +async function writeReference( + packageDir: string, + kind: "bundled" | "optional", + category: string, + slug: string, + descriptor: Record, +) { + const skillDir = path.join(packageDir, "catalog", kind, category, slug); + await fs.mkdir(skillDir, { recursive: true }); + await fs.writeFile( + path.join(skillDir, "catalog-ref.json"), + `${JSON.stringify(descriptor, null, 2)}\n`, + "utf8", + ); +} diff --git a/packages/skills-catalog/src/catalog-builder.ts b/packages/skills-catalog/src/catalog-builder.ts index ac0502fe..1c9910ae 100644 --- a/packages/skills-catalog/src/catalog-builder.ts +++ b/packages/skills-catalog/src/catalog-builder.ts @@ -4,6 +4,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { asBoolean, + isPlainRecord, asString, asStringArray, parseFrontmatterMarkdown, @@ -14,23 +15,54 @@ import type { CatalogSkillFile, CatalogSkillFileKind, CatalogSkillKind, + CatalogSkillSource, CatalogTrustLevel, } from "./types.js"; const CATALOG_PACKAGE_NAME = "@paperclipai/skills-catalog"; const CATALOG_SCHEMA_VERSION = 1; const SKILL_ENTRYPOINT = "SKILL.md"; +const CATALOG_REFERENCE_FILE = "catalog-ref.json"; const MAX_CATALOG_FILE_BYTES = 1024 * 1024; const SLUG_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; const CATALOG_KINDS = new Set(["bundled", "optional"]); -interface SkillCandidate { +interface BaseSkillCandidate { kind: CatalogSkillKind; category: string; slug: string; absolutePath: string; } +type SkillCandidate = + | (BaseSkillCandidate & { source: "local" }) + | (BaseSkillCandidate & { source: "reference"; descriptorPath: string }); + +interface ReferencedGitHubSourceDescriptor { + type: "github"; + hostname?: string; + owner: string; + repo: string; + ref: string; + commit: string; + path: string; +} + +interface ReferencedSkillDescriptor { + source: ReferencedGitHubSourceDescriptor; + files?: string[]; + defaultInstall?: boolean; + recommendedForRoles?: string[]; + requires?: string[]; + tags?: string[]; +} + +interface GitHubTreeEntry { + path: string; + type: "blob" | "tree" | string; + size?: number; +} + interface BuildCatalogManifestOptions { packageDir: string; generatedAt?: string; @@ -171,11 +203,23 @@ async function discoverSkillCandidates(packageDir: string, errors: string[]) { if (!slugEntry.isDirectory()) continue; const slug = slugEntry.name; const skillDir = path.join(categoryDir, slug); - if (!existsSync(path.join(skillDir, SKILL_ENTRYPOINT))) { - errors.push(`${relativePackagePath(packageDir, skillDir)} is missing SKILL.md.`); + const hasLocalSkill = existsSync(path.join(skillDir, SKILL_ENTRYPOINT)); + const descriptorPath = path.join(skillDir, CATALOG_REFERENCE_FILE); + const hasReference = existsSync(descriptorPath); + + if (hasLocalSkill && hasReference) { + errors.push(`${relativePackagePath(packageDir, skillDir)} must contain either SKILL.md or ${CATALOG_REFERENCE_FILE}, not both.`); continue; } - candidates.push({ kind, category, slug, absolutePath: skillDir }); + if (!hasLocalSkill && !hasReference) { + errors.push(`${relativePackagePath(packageDir, skillDir)} is missing SKILL.md or ${CATALOG_REFERENCE_FILE}.`); + continue; + } + candidates.push( + hasReference + ? { kind, category, slug, absolutePath: skillDir, source: "reference", descriptorPath } + : { kind, category, slug, absolutePath: skillDir, source: "local" }, + ); } } } @@ -191,13 +235,13 @@ async function collectMisplacedSkillFiles(catalogDir: string, errors: string[]) await visit(absolutePath); continue; } - if (entry.name !== SKILL_ENTRYPOINT) continue; + if (entry.name !== SKILL_ENTRYPOINT && entry.name !== CATALOG_REFERENCE_FILE) continue; const relativePath = toPosixPath(path.relative(catalogDir, absolutePath)); const parts = relativePath.split("/"); const kind = parts[0]; if (parts.length !== 4 || !CATALOG_KINDS.has(kind as CatalogSkillKind)) { - errors.push(`catalog/${relativePath} is not under catalog////SKILL.md.`); + errors.push(`catalog/${relativePath} is not under catalog////{SKILL.md,${CATALOG_REFERENCE_FILE}}.`); } } } @@ -210,6 +254,10 @@ async function buildCatalogSkill( candidate: SkillCandidate, errors: string[], ): Promise { + if (candidate.source === "reference") { + return buildReferencedCatalogSkill(packageDir, candidate, errors); + } + const prefix = relativePackagePath(packageDir, candidate.absolutePath); validateSlug("category", candidate.category, prefix, errors); validateSlug("slug", candidate.slug, prefix, errors); @@ -268,6 +316,268 @@ async function buildCatalogSkill( }; } +async function buildReferencedCatalogSkill( + packageDir: string, + candidate: Extract, + errors: string[], +): Promise { + const prefix = relativePackagePath(packageDir, candidate.absolutePath); + validateSlug("category", candidate.category, prefix, errors); + validateSlug("slug", candidate.slug, prefix, errors); + + const descriptor = await readReferencedSkillDescriptor(candidate.descriptorPath, prefix, errors); + if (!descriptor) return null; + + const id = `paperclipai:${candidate.kind}:${candidate.category}:${candidate.slug}`; + const key = `paperclipai/${candidate.kind}/${candidate.category}/${candidate.slug}`; + const source = buildCatalogSkillSource(descriptor.source, errors, `${prefix}/${CATALOG_REFERENCE_FILE}`); + if (!source) return null; + + const files = await collectReferencedSkillFiles(source, descriptor.files ?? [SKILL_ENTRYPOINT], prefix, errors); + const skillMarkdown = await readReferencedFileText(source, SKILL_ENTRYPOINT, prefix, errors); + if (!skillMarkdown) return null; + + const parsed = parseFrontmatterMarkdown(skillMarkdown); + if (!parsed.hasFrontmatter) { + errors.push(`${source.url}/${SKILL_ENTRYPOINT} must start with YAML frontmatter.`); + } + + const name = asString(parsed.frontmatter.name); + if (!name) errors.push(`${source.url}/${SKILL_ENTRYPOINT} frontmatter must include name.`); + + const description = asString(parsed.frontmatter.description); + if (!description) errors.push(`${source.url}/${SKILL_ENTRYPOINT} frontmatter must include description.`); + + const explicitKey = asString(parsed.frontmatter.key); + if (explicitKey && explicitKey !== key) { + errors.push(`${source.url}/${SKILL_ENTRYPOINT} key must be ${key}.`); + } + + const explicitSlug = asString(parsed.frontmatter.slug); + if (explicitSlug && explicitSlug !== candidate.slug) { + errors.push(`${source.url}/${SKILL_ENTRYPOINT} slug must be ${candidate.slug}.`); + } + + const defaultInstall = asBoolean(descriptor.defaultInstall) ?? false; + const recommendedForRoles = readStringArrayField(descriptor.recommendedForRoles, "recommendedForRoles", prefix, errors); + const requires = readStringArrayField(descriptor.requires, "requires", prefix, errors); + const tags = readStringArrayField(descriptor.tags, "tags", prefix, errors); + + if (!files.some((file) => file.path === SKILL_ENTRYPOINT && file.kind === "skill")) { + errors.push(`${prefix} referenced inventory does not contain SKILL.md.`); + } + if (!name || !description) return null; + + return { + id, + key, + kind: candidate.kind, + category: candidate.category, + slug: candidate.slug, + name, + description, + path: toPosixPath(path.relative(packageDir, candidate.absolutePath)), + entrypoint: SKILL_ENTRYPOINT, + trustLevel: deriveTrustLevel(files), + compatibility: "compatible", + defaultInstall, + recommendedForRoles, + requires, + tags, + files, + contentHash: buildContentHash(files), + source, + }; +} + +async function readReferencedSkillDescriptor( + descriptorPath: string, + prefix: string, + errors: string[], +): Promise { + let raw: unknown; + try { + raw = JSON.parse(await fs.readFile(descriptorPath, "utf8")); + } catch (error) { + errors.push(`${prefix}/${CATALOG_REFERENCE_FILE} is missing or invalid JSON: ${errorMessage(error)}`); + return null; + } + + if (!isPlainRecord(raw) || !isPlainRecord(raw.source)) { + errors.push(`${prefix}/${CATALOG_REFERENCE_FILE} must include a source object.`); + return null; + } + const sourceRaw = raw.source; + if (asString(sourceRaw.type) !== "github") { + errors.push(`${prefix}/${CATALOG_REFERENCE_FILE} source.type must be "github".`); + return null; + } + + const owner = asString(sourceRaw.owner); + const repo = asString(sourceRaw.repo); + const ref = asString(sourceRaw.ref); + const commit = asString(sourceRaw.commit); + const sourcePath = asString(sourceRaw.path); + if (!owner || !repo || !ref || !commit || sourcePath === null) { + errors.push(`${prefix}/${CATALOG_REFERENCE_FILE} GitHub source must include owner, repo, ref, commit, and path.`); + return null; + } + + const descriptor: ReferencedSkillDescriptor = { + source: { + type: "github", + hostname: asString(sourceRaw.hostname) ?? "github.com", + owner, + repo, + ref, + commit, + path: sourcePath, + }, + defaultInstall: asBoolean(raw.defaultInstall) ?? false, + files: asStringArray(raw.files ?? undefined) ?? undefined, + recommendedForRoles: asStringArray(raw.recommendedForRoles ?? undefined) ?? undefined, + requires: asStringArray(raw.requires ?? undefined) ?? undefined, + tags: asStringArray(raw.tags ?? undefined) ?? undefined, + }; + + if (raw.files !== undefined && !descriptor.files) errors.push(`${prefix}/${CATALOG_REFERENCE_FILE} files must be an array of strings.`); + if (raw.recommendedForRoles !== undefined && !descriptor.recommendedForRoles) errors.push(`${prefix}/${CATALOG_REFERENCE_FILE} recommendedForRoles must be an array of strings.`); + if (raw.requires !== undefined && !descriptor.requires) errors.push(`${prefix}/${CATALOG_REFERENCE_FILE} requires must be an array of strings.`); + if (raw.tags !== undefined && !descriptor.tags) errors.push(`${prefix}/${CATALOG_REFERENCE_FILE} tags must be an array of strings.`); + + return descriptor; +} + +function buildCatalogSkillSource( + descriptor: ReferencedGitHubSourceDescriptor, + errors: string[], + prefix: string, +): CatalogSkillSource | null { + if (!/^[0-9a-f]{40}$/i.test(descriptor.commit)) { + errors.push(`${prefix} source.commit must be a 40-character Git commit SHA.`); + } + const sourcePath = normalizeReferencedPath(descriptor.path); + if (sourcePath === null) { + errors.push(`${prefix} source.path must be a portable path within the repository.`); + } + const hostname = descriptor.hostname ?? "github.com"; + const url = `https://${hostname}/${descriptor.owner}/${descriptor.repo}/tree/${descriptor.ref}/${sourcePath ?? ""}`.replace(/\/$/, ""); + if (errors.length > 0 && (!/^[0-9a-f]{40}$/i.test(descriptor.commit) || sourcePath === null)) return null; + return { + type: "github", + hostname, + owner: descriptor.owner, + repo: descriptor.repo, + ref: descriptor.ref, + commit: descriptor.commit, + path: sourcePath ?? "", + url, + }; +} + +async function collectReferencedSkillFiles( + source: CatalogSkillSource, + includePatterns: string[], + prefix: string, + errors: string[], +): Promise { + const tree = await fetchGitHubTree(source, prefix, errors); + const sourceRoot = source.path ? `${source.path}/` : ""; + const normalizedPatterns: string[] = []; + for (const pattern of includePatterns) { + const normalizedPattern = normalizeReferencedPath(pattern); + if (normalizedPattern === null) { + errors.push(`${prefix} referenced include path is invalid: ${pattern}`); + continue; + } + if (normalizedPattern) normalizedPatterns.push(normalizedPattern); + } + const files: CatalogSkillFile[] = []; + + for (const entry of tree) { + if (entry.type !== "blob") continue; + if (!entry.path.startsWith(sourceRoot)) continue; + const relativePath = entry.path.slice(sourceRoot.length); + if (!normalizedPatterns.some((pattern) => referencedPathMatches(relativePath, pattern))) continue; + if ((entry.size ?? 0) > MAX_CATALOG_FILE_BYTES) { + errors.push(`${prefix}/${relativePath} exceeds ${MAX_CATALOG_FILE_BYTES} bytes.`); + continue; + } + + const bytes = await fetchReferencedFileBytes(source, relativePath, prefix, errors); + if (!bytes) continue; + files.push({ + path: relativePath, + kind: classifyCatalogFile(relativePath), + sizeBytes: bytes.byteLength, + sha256: sha256(bytes), + }); + } + + files.sort((a, b) => { + if (a.path === SKILL_ENTRYPOINT) return -1; + if (b.path === SKILL_ENTRYPOINT) return 1; + return a.path.localeCompare(b.path); + }); + return files; +} + +async function fetchGitHubTree( + source: CatalogSkillSource, + prefix: string, + errors: string[], +): Promise { + const url = `${githubApiBase(source.hostname)}/repos/${source.owner}/${source.repo}/git/trees/${source.commit}?recursive=1`; + try { + const response = await fetch(url, { headers: { accept: "application/vnd.github+json" } }); + if (!response.ok) { + errors.push(`${prefix} failed to fetch GitHub tree: HTTP ${response.status}.`); + return []; + } + const body = await response.json() as { tree?: GitHubTreeEntry[]; truncated?: boolean }; + if (body.truncated) errors.push(`${prefix} GitHub tree response was truncated.`); + return Array.isArray(body.tree) ? body.tree : []; + } catch (error) { + errors.push(`${prefix} failed to fetch GitHub tree: ${errorMessage(error)}.`); + return []; + } +} + +async function readReferencedFileText( + source: CatalogSkillSource, + relativePath: string, + prefix: string, + errors: string[], +) { + const bytes = await fetchReferencedFileBytes(source, relativePath, prefix, errors); + return bytes ? bytes.toString("utf8") : null; +} + +async function fetchReferencedFileBytes( + source: CatalogSkillSource, + relativePath: string, + prefix: string, + errors: string[], +): Promise { + const normalizedPath = normalizeReferencedPath(relativePath); + if (!normalizedPath) { + errors.push(`${prefix} referenced file path is invalid: ${relativePath}`); + return null; + } + const url = rawGitHubUrl(source, normalizedPath); + try { + const response = await fetch(url); + if (!response.ok) { + errors.push(`${prefix}/${normalizedPath} failed to fetch pinned GitHub file: HTTP ${response.status}.`); + return null; + } + return Buffer.from(await response.arrayBuffer()); + } catch (error) { + errors.push(`${prefix}/${normalizedPath} failed to fetch pinned GitHub file: ${errorMessage(error)}.`); + return null; + } +} + async function collectSkillFiles( packageDir: string, skillDir: string, @@ -433,6 +743,40 @@ function toPosixPath(input: string) { return input.split(path.sep).join("/"); } +function normalizeReferencedPath(input: string) { + const normalized = input.replace(/\\/g, "/").replace(/^\/+/, "").replace(/\/+$/, ""); + if (normalized === "") return ""; + const parts = normalized.split("/"); + if (parts.includes("") || parts.includes(".") || parts.includes("..") || path.posix.isAbsolute(normalized)) { + return null; + } + return normalized; +} + +function referencedPathMatches(relativePath: string, pattern: string) { + if (pattern.endsWith("/**")) { + const directory = pattern.slice(0, -3); + return relativePath === directory || relativePath.startsWith(`${directory}/`); + } + return relativePath === pattern; +} + +function githubApiBase(hostname: string) { + const normalized = hostname.toLowerCase(); + return normalized === "github.com" || normalized === "www.github.com" + ? "https://api.github.com" + : `https://${hostname}/api/v3`; +} + +function rawGitHubUrl(source: CatalogSkillSource, relativePath: string) { + const fullPath = source.path ? `${source.path}/${relativePath}` : relativePath; + const encodedPath = fullPath.split("/").map((segment) => encodeURIComponent(segment)).join("/"); + const normalized = source.hostname.toLowerCase(); + return normalized === "github.com" || normalized === "www.github.com" + ? `https://raw.githubusercontent.com/${source.owner}/${source.repo}/${source.commit}/${encodedPath}` + : `https://${source.hostname}/raw/${source.owner}/${source.repo}/${source.commit}/${encodedPath}`; +} + function isPathInside(parent: string, child: string) { const relativePath = path.relative(parent, child); return relativePath === "" || (!relativePath.startsWith("..") && !path.isAbsolute(relativePath)); diff --git a/packages/skills-catalog/src/index.ts b/packages/skills-catalog/src/index.ts index 70fefd2d..ff5baee1 100644 --- a/packages/skills-catalog/src/index.ts +++ b/packages/skills-catalog/src/index.ts @@ -7,7 +7,9 @@ export type { CatalogSkill, CatalogSkillFile, CatalogSkillFileKind, + CatalogSkillGitHubSource, CatalogSkillKind, + CatalogSkillSource, CatalogTrustLevel, CatalogValidationResult, } from "./types.js"; diff --git a/packages/skills-catalog/src/shipped-catalog.test.ts b/packages/skills-catalog/src/shipped-catalog.test.ts index 867f71e8..d77f4c9a 100644 --- a/packages/skills-catalog/src/shipped-catalog.test.ts +++ b/packages/skills-catalog/src/shipped-catalog.test.ts @@ -1,6 +1,5 @@ import { describe, expect, it } from "vitest"; import { catalogManifest, catalogSkills, resolveCatalogSkillRef } from "./index.js"; -import type { CatalogSkill } from "./types.js"; const EXPECTED_BUNDLED_KEYS = [ "paperclipai/bundled/docs/doc-maintenance", @@ -15,6 +14,7 @@ const EXPECTED_OPTIONAL_KEYS = [ "paperclipai/optional/browser/agent-browser", "paperclipai/optional/content/release-announcement", "paperclipai/optional/product/design-critique", + "paperclipai/optional/research/last30days", ]; describe("shipped skills catalog", () => { @@ -32,12 +32,14 @@ describe("shipped skills catalog", () => { expect(optionalKeys).toEqual(EXPECTED_OPTIONAL_KEYS); }); - it("keeps every shipped skill free of executable scripts until script-bearing skills clear security review", () => { - // The real install-time security boundary (server assertCatalogSkillInstallable) blocks - // only "scripts_executables". Static assets (svg/html templates, e.g. the wireframe skill) - // carry the "assets" trust level and are installable, so they are allowed in the catalog. + it("keeps script-bearing shipped skills explicit so install stays audit-gated", () => { + // The real install-time security boundary audits materialized bytes and blocks + // hard-stop findings. Static assets (svg/html templates, e.g. the wireframe skill) + // carry the "assets" trust level and are installable. const scriptBearing = catalogSkills.filter((skill) => skill.trustLevel === "scripts_executables"); - expect(scriptBearing, formatViolations("script-bearing skills require security review", scriptBearing)).toEqual([]); + expect(scriptBearing.map((skill) => skill.key)).toEqual([ + "paperclipai/optional/research/last30days", + ]); }); it("populates browse/search-relevant fields for every shipped skill", () => { @@ -86,9 +88,3 @@ describe("shipped skills catalog", () => { expect(resolveCatalogSkillRef(sample.slug)).toMatchObject({ key: sample.key }); }); }); - -function formatViolations(label: string, skills: CatalogSkill[]) { - if (skills.length === 0) return label; - const detail = skills.map((skill) => `${skill.key} (${skill.trustLevel})`).join(", "); - return `${label}: ${detail}`; -} diff --git a/packages/skills-catalog/src/types.ts b/packages/skills-catalog/src/types.ts index 0985974e..4921f808 100644 --- a/packages/skills-catalog/src/types.ts +++ b/packages/skills-catalog/src/types.ts @@ -13,6 +13,19 @@ export interface CatalogSkillFile { sha256: string; } +export interface CatalogSkillGitHubSource { + type: "github"; + hostname: string; + owner: string; + repo: string; + ref: string; + commit: string; + path: string; + url: string; +} + +export type CatalogSkillSource = CatalogSkillGitHubSource; + export interface CatalogSkill { id: string; key: string; @@ -31,6 +44,7 @@ export interface CatalogSkill { tags: string[]; files: CatalogSkillFile[]; contentHash: string; + source?: CatalogSkillSource; } export interface CatalogManifest { diff --git a/server/src/__tests__/company-skills-catalog-service.test.ts b/server/src/__tests__/company-skills-catalog-service.test.ts index 4fcfdde0..c2fbe601 100644 --- a/server/src/__tests__/company-skills-catalog-service.test.ts +++ b/server/src/__tests__/company-skills-catalog-service.test.ts @@ -223,6 +223,47 @@ describeEmbeddedPostgres("companySkillService.installFromCatalog", () => { }); }); + it("installs script-bearing catalog skills when materialized bytes pass the static audit", async () => { + const script = "print('safe')\n"; + const scriptFiles: CatalogSkillFile[] = [ + ...sampleFiles, + { path: "scripts/run.py", kind: "script", sizeBytes: Buffer.byteLength(script), sha256: sha256(script) }, + ]; + const scriptCatalogSkill: CatalogSkill = { + ...sampleCatalogSkill, + trustLevel: "scripts_executables", + files: scriptFiles, + contentHash: contentHash(scriptFiles), + }; + mockCatalogService.getCatalogSkillOrThrow.mockReturnValue(scriptCatalogSkill); + mockCatalogService.copyCatalogSkillFile.mockImplementation(async (_ref: string, filePath: string, targetPath: string) => { + if (filePath === "scripts/run.py") { + await fs.writeFile(targetPath, script, "utf8"); + return; + } + const content = filePath === "SKILL.md" ? sampleSkillMarkdown : sampleReferenceMarkdown; + await fs.writeFile(targetPath, content, "utf8"); + }); + const companyId = await createCompany(); + + const result = await svc.installFromCatalog(companyId, { + catalogSkillId: scriptCatalogSkill.id, + }); + + expect(result.action).toBe("created"); + expect(result.skill).toMatchObject({ + trustLevel: "scripts_executables", + metadata: expect.objectContaining({ + auditVerdict: "warning", + auditCodes: expect.arrayContaining(["script_trust"]), + }), + }); + expect(result.warnings).toEqual(expect.arrayContaining([ + "Skill includes a script file.", + ])); + await expect(fs.readFile(path.join(result.skill.sourceLocator!, "scripts/run.py"), "utf8")).resolves.toBe(script); + }); + it("restores portable catalog provenance when importing packaged skills", async () => { const companyId = await createCompany(); const importedFiles = { diff --git a/server/src/__tests__/skills-catalog-service.test.ts b/server/src/__tests__/skills-catalog-service.test.ts index 24a5664f..44fed3cc 100644 --- a/server/src/__tests__/skills-catalog-service.test.ts +++ b/server/src/__tests__/skills-catalog-service.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { CatalogSkill } from "@paperclipai/shared"; @@ -42,6 +43,10 @@ function catalogSkill(slug: string, name = slug): CatalogSkill { }; } +function sha256(value: string | Buffer) { + return createHash("sha256").update(value).digest("hex"); +} + function manifest(skills: CatalogSkill[], packageVersion = "0.3.1") { return JSON.stringify({ schemaVersion: 1, @@ -59,6 +64,7 @@ describe("skills catalog service", () => { beforeEach(() => { vi.resetModules(); vi.clearAllMocks(); + vi.unstubAllGlobals(); manifestJson = manifest([catalogSkill("old-skill", "Old Skill")]); manifestMtimeMs = 1; mockExistsSync.mockReturnValue(true); @@ -110,4 +116,38 @@ describe("skills catalog service", () => { }); expect(mockReadFile).not.toHaveBeenCalled(); }); + + it("reads referenced GitHub catalog files from pinned raw bytes", async () => { + const markdown = "---\nname: remote\n---\n\n# Remote\n"; + const remoteSkill = catalogSkill("remote-skill", "Remote Skill"); + remoteSkill.source = { + type: "github", + hostname: "github.com", + owner: "example", + repo: "remote-skill", + ref: "v1.0.0", + commit: "0123456789abcdef0123456789abcdef01234567", + path: "skills/remote", + url: "https://github.com/example/remote-skill/tree/v1.0.0/skills/remote", + }; + remoteSkill.files = [ + { path: "SKILL.md", kind: "skill", sizeBytes: Buffer.byteLength(markdown), sha256: sha256(markdown) }, + ]; + manifestJson = manifest([remoteSkill]); + const fetchMock = vi.fn(async (url: string) => { + expect(url).toBe("https://raw.githubusercontent.com/example/remote-skill/0123456789abcdef0123456789abcdef01234567/skills/remote/SKILL.md"); + return new Response(markdown, { status: 200 }); + }); + vi.stubGlobal("fetch", fetchMock); + const service = await import("../services/skills-catalog.js"); + + await expect(service.readCatalogSkillFile(remoteSkill.id, "SKILL.md")).resolves.toMatchObject({ + catalogSkillId: remoteSkill.id, + path: "SKILL.md", + content: markdown, + markdown: true, + }); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(mockReadFile).not.toHaveBeenCalled(); + }); }); diff --git a/server/src/services/company-skills.ts b/server/src/services/company-skills.ts index d78d274f..c25e29d5 100644 --- a/server/src/services/company-skills.ts +++ b/server/src/services/company-skills.ts @@ -44,7 +44,6 @@ import { copyCatalogSkillFile, getCatalogPackageMetadata, getCatalogSkillOrThrow, - readCatalogSkillFile, resolveCatalogSkillReference, } from "./skills-catalog.js"; import { @@ -2452,7 +2451,7 @@ export function companySkillService(db: Db) { buildSkillRuntimeName(catalogSkill.key, skill.slug), ); await copySkillDirectory(originSnapshotLocator, materializedDir); - const markdown = (await readCatalogSkillFile(catalogSkill.id, catalogSkill.entrypoint)).content; + const markdown = await fs.readFile(path.join(originSnapshotLocator, catalogSkill.entrypoint), "utf8"); const nextMetadata = buildCatalogSkillMetadata(catalogSkill, skill, originSnapshotLocator); const nextValues = { name: catalogSkill.name, @@ -2942,6 +2941,7 @@ export function companySkillService(db: Db) { catalogKind: catalogSkill.kind, catalogCategory: catalogSkill.category, catalogPath: catalogSkill.path, + catalogSource: catalogSkill.source ?? null, packageName: packageMetadata.packageName, packageVersion: packageMetadata.packageVersion, originHash: catalogSkill.contentHash, @@ -2956,11 +2956,36 @@ export function companySkillService(db: Db) { if (catalogSkill.compatibility !== "compatible") { throw unprocessable(`Catalog skill ${catalogSkill.id} is not compatible.`); } - if (catalogSkill.trustLevel === "scripts_executables") { - throw unprocessable( - "Catalog skill contains executable scripts and cannot be force-installed until security review semantics allow it.", - ); - } + } + + async function auditCatalogSkillSnapshot( + companyId: string, + catalogSkill: CatalogSkill, + slug: string, + sourceDir: string, + ) { + return auditInstalledSkillBytes({ + id: randomUUID(), + companyId, + key: catalogSkill.key, + slug, + name: catalogSkill.name, + description: catalogSkill.description, + markdown: "", + sourceType: "catalog", + sourceLocator: sourceDir, + sourceRef: catalogSkill.contentHash, + trustLevel: catalogSkill.trustLevel, + compatibility: catalogSkill.compatibility, + fileInventory: catalogSkill.files.map((entry) => ({ path: entry.path, kind: entry.kind })), + metadata: { + sourceKind: "catalog", + catalogId: catalogSkill.id, + originHash: catalogSkill.contentHash, + }, + createdAt: new Date(), + updatedAt: new Date(), + }); } async function installFromCatalog( @@ -3021,9 +3046,27 @@ export function companySkillService(db: Db) { } } - const materializedDir = await materializeCatalogManifestSkillFiles(companyId, catalogSkill, slug); - const originSnapshotLocator = await materializeCatalogOriginSnapshot(companyId, catalogSkill, slug); - const markdown = (await readCatalogSkillFile(catalogSkill.id, catalogSkill.entrypoint)).content; + let materializedDir: string | null = null; + let originSnapshotLocator: string | null = null; + try { + originSnapshotLocator = await materializeCatalogOriginSnapshot(companyId, catalogSkill, slug); + const candidateAudit = await auditCatalogSkillSnapshot(companyId, catalogSkill, slug, originSnapshotLocator); + if (candidateAudit.verdict === "fail") { + throw unprocessable("Catalog install is blocked by hard-stop audit findings.", { + updateHoldReason: "audit_hard_stop", + audit: candidateAudit, + }); + } + materializedDir = await materializeCatalogManifestSkillFiles(companyId, catalogSkill, slug); + } catch (error) { + if (materializedDir) await fs.rm(materializedDir, { recursive: true, force: true }).catch(() => undefined); + if (originSnapshotLocator) await fs.rm(originSnapshotLocator, { recursive: true, force: true }).catch(() => undefined); + throw error; + } + if (!materializedDir || !originSnapshotLocator) { + throw unprocessable("Catalog install did not materialize pinned files."); + } + const markdown = await fs.readFile(path.join(originSnapshotLocator, catalogSkill.entrypoint), "utf8"); const metadata = buildCatalogSkillMetadata(catalogSkill, existingByKey, originSnapshotLocator); const values = { companyId, diff --git a/server/src/services/skills-catalog.ts b/server/src/services/skills-catalog.ts index 26a4900f..d8ce8f9c 100644 --- a/server/src/services/skills-catalog.ts +++ b/server/src/services/skills-catalog.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { existsSync, readFileSync, statSync } from "node:fs"; import { promises as fs } from "node:fs"; import path from "node:path"; @@ -6,8 +7,10 @@ import type { CatalogSkill, CatalogSkillFileDetail, CatalogSkillListQuery, + CatalogSkillSource, } from "@paperclipai/shared"; -import { HttpError, conflict, notFound } from "../errors.js"; +import { HttpError, conflict, notFound, unprocessable } from "../errors.js"; +import { ghFetch, resolveRawGitHubUrl } from "./github-fetch.js"; import { normalizePortablePath } from "./portable-path.js"; interface CatalogManifestFile { @@ -93,6 +96,60 @@ function resolveCatalogPackageRoot() { return catalogPackageRoot; } +function sourceRootPath(source: CatalogSkillSource) { + return source.path ? normalizePortablePath(source.path) : ""; +} + +function resolveCatalogSourcePath(source: CatalogSkillSource, relativePath: string) { + const sourceRoot = sourceRootPath(source); + return sourceRoot ? `${sourceRoot}/${relativePath}` : relativePath; +} + +async function fetchCatalogSourceFile( + skill: CatalogSkill, + relativePath: string, +): Promise { + const source = skill.source; + if (!source) { + const packageRoot = resolveCatalogPackageRoot(); + const absolutePath = path.resolve(packageRoot, skill.path, relativePath); + const skillRoot = path.resolve(packageRoot, skill.path); + if (absolutePath !== skillRoot && !absolutePath.startsWith(`${skillRoot}${path.sep}`)) { + throw notFound("Catalog skill file not found"); + } + return fs.readFile(absolutePath); + } + + if (source.type !== "github") { + throw unprocessable(`Unsupported catalog source type: ${(source as { type: string }).type}`); + } + + const sourcePath = resolveCatalogSourcePath(source, relativePath); + const url = resolveRawGitHubUrl(source.hostname, source.owner, source.repo, source.commit, sourcePath); + const response = await ghFetch(url); + if (!response.ok) { + throw unprocessable(`Failed to fetch pinned catalog file ${sourcePath} from ${source.owner}/${source.repo}@${source.commit}: HTTP ${response.status}`); + } + return Buffer.from(await response.arrayBuffer()); +} + +async function readCatalogFileBytes( + skill: CatalogSkill, + relativePath: string, +): Promise { + const fileEntry = skill.files.find((entry) => entry.path === relativePath); + if (!fileEntry) { + throw notFound("Catalog skill file not found"); + } + + const bytes = await fetchCatalogSourceFile(skill, relativePath); + const actualSha = createHash("sha256").update(bytes).digest("hex"); + if (actualSha !== fileEntry.sha256) { + throw unprocessable(`Pinned catalog file hash mismatch for ${skill.id}:${relativePath}.`); + } + return bytes; +} + function searchText(skill: CatalogSkill) { return [ skill.id, @@ -152,18 +209,11 @@ export async function readCatalogSkillFile( throw notFound("Catalog skill file not found"); } - const packageRoot = resolveCatalogPackageRoot(); - const absolutePath = path.resolve(packageRoot, skill.path, normalizedPath); - const skillRoot = path.resolve(packageRoot, skill.path); - if (absolutePath !== skillRoot && !absolutePath.startsWith(`${skillRoot}${path.sep}`)) { - throw notFound("Catalog skill file not found"); - } - if (fileEntry.kind === "asset") { throw new HttpError(415, "Catalog asset previews are not supported."); } - const content = await fs.readFile(absolutePath, "utf8"); + const content = (await readCatalogFileBytes(skill, normalizedPath)).toString("utf8"); return { catalogSkillId: skill.id, path: normalizedPath, @@ -182,14 +232,7 @@ export async function copyCatalogSkillFile(reference: string, relativePath: stri throw notFound("Catalog skill file not found"); } - const packageRoot = resolveCatalogPackageRoot(); - const absolutePath = path.resolve(packageRoot, skill.path, normalizedPath); - const skillRoot = path.resolve(packageRoot, skill.path); - if (absolutePath !== skillRoot && !absolutePath.startsWith(`${skillRoot}${path.sep}`)) { - throw notFound("Catalog skill file not found"); - } - - await fs.copyFile(absolutePath, targetPath); + await fs.writeFile(targetPath, await readCatalogFileBytes(skill, normalizedPath)); } export function getCatalogPackageMetadata() {