fix: parse YAML block scalar skill descriptions (#5046)
## Thinking Path > - Paperclip is the open source control plane teams use to manage AI agents for work. > - Company skills are imported from `SKILL.md` files and rely on YAML frontmatter to describe what each skill does. > - Multi-line descriptions commonly use YAML block scalars (`>` and `|`), but the broken parser path behind #4989 reduced those descriptions to a literal `>` or `|`. > - The earliest contributor fix for that bug was PR #5046, so this branch keeps that PR as the canonical merge target instead of replacing it. > - Follow-up work from #5071 and #8258 was then transplanted onto this earlier branch so the final PR preserves contributor credit while still shipping the strongest complete fix. > - The resulting change fixes block-scalar parsing in the shared frontmatter path, aligns server company-skill imports with that shared parser, and prevents already-stale stored markers from rendering as junk in the UI. ## Linked Issues or Issue Description - Fixes #4989 - Refs #2863 - Refs #788 - Related superseded PRs: #5071, #8258 ## What Changed - Kept the original PR #5046 server-side company-skill fix and regression coverage as the base branch history. - Added the missing YAML chomping and indicator hardening explored further in #5071. - Moved frontmatter parsing to the shared parser path so `packages/shared`, `packages/skills-catalog`, and server company-skill imports stay aligned. - Added UI summary sanitization and fallback handling so stale stored `>` / `|` values no longer render as visible junk in company-skill cards. - Added regression coverage for shared frontmatter parsing, skills-catalog parsing, company-skill imports, and stale-summary fallback behavior. ## Verification - Passed locally: `pnpm exec vitest run packages/shared/src/frontmatter.test.ts packages/skills-catalog/src/frontmatter.test.ts server/src/__tests__/company-skills.test.ts ui/src/lib/company-skill-summary.test.ts` - Passed locally: `pnpm --filter @paperclipai/shared typecheck` - Passed locally: `pnpm --filter @paperclipai/skills-catalog typecheck` - Not fully runnable in this worktree: `pnpm --filter @paperclipai/server typecheck` currently fails in `packages/plugins/sdk` before reaching server code because local workspace `node_modules` type deps are missing (`TS2688` for `node` / `react`). - GitHub Actions / PR checks are rerunning on PR #5046 head `005290b7557725abf748d00f36dd24ea0d919aba`. ## Risks - Medium-low risk: the fix now touches shared parser code, server company-skill imports, and UI fallback display rather than only the server import path. - The parser is still intentionally narrower than a full YAML implementation; this change focuses on block-scalar correctness and the stale-description rendering path relevant to #4989 / #2863. - This branch intentionally supersedes narrower overlapping work from #5071 and duplicate work from #8258 once the survivor PR is green. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex / GPT-5-based coding agent with local shell and code-editing tools enabled. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Devin Foley <devin@paperclip.ing>
This commit is contained in:
MrBob
@@ -0,0 +1,101 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseFrontmatterMarkdown } from "./frontmatter.js";
|
||||
|
||||
describe("parseFrontmatterMarkdown", () => {
|
||||
it("parses folded and literal YAML block scalars", () => {
|
||||
const folded = parseFrontmatterMarkdown([
|
||||
"---",
|
||||
"name: Folded",
|
||||
"description: >",
|
||||
" First line",
|
||||
" second line",
|
||||
"",
|
||||
" Third paragraph",
|
||||
"---",
|
||||
"",
|
||||
"Body",
|
||||
].join("\n"));
|
||||
|
||||
expect(folded.frontmatter.description).toBe("First line second line\n\nThird paragraph\n");
|
||||
|
||||
const literal = parseFrontmatterMarkdown([
|
||||
"---",
|
||||
"name: Literal",
|
||||
"description: |",
|
||||
" First line",
|
||||
" second line",
|
||||
"---",
|
||||
"",
|
||||
"Body",
|
||||
].join("\n"));
|
||||
|
||||
expect(literal.frontmatter.description).toBe("First line\nsecond line\n");
|
||||
});
|
||||
|
||||
it("respects block-scalar chomping indicators", () => {
|
||||
const foldedStrip = parseFrontmatterMarkdown([
|
||||
"---",
|
||||
"description: >-",
|
||||
" First line",
|
||||
" second line",
|
||||
"",
|
||||
" Third paragraph",
|
||||
"---",
|
||||
"",
|
||||
"Body",
|
||||
].join("\n"));
|
||||
|
||||
expect(foldedStrip.frontmatter.description).toBe("First line second line\n\nThird paragraph");
|
||||
|
||||
const literalKeep = parseFrontmatterMarkdown([
|
||||
"---",
|
||||
"description: |+",
|
||||
" First line",
|
||||
" second line",
|
||||
"",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
"Body",
|
||||
].join("\n"));
|
||||
|
||||
expect(literalKeep.frontmatter.description).toBe("First line\nsecond line\n\n");
|
||||
});
|
||||
|
||||
it("parses inline object array items nested under frontmatter keys", () => {
|
||||
const parsed = parseFrontmatterMarkdown([
|
||||
"---",
|
||||
"metadata:",
|
||||
" sources:",
|
||||
" - kind: github-dir",
|
||||
" repo: paperclipai/paperclip",
|
||||
" path: skills/paperclip",
|
||||
"---",
|
||||
"",
|
||||
"Body",
|
||||
].join("\n"));
|
||||
|
||||
expect(parsed.frontmatter).toMatchObject({
|
||||
metadata: {
|
||||
sources: [
|
||||
{
|
||||
kind: "github-dir",
|
||||
repo: "paperclipai/paperclip",
|
||||
path: "skills/paperclip",
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("does not treat trailing-dot decimals as numbers", () => {
|
||||
const parsed = parseFrontmatterMarkdown([
|
||||
"---",
|
||||
"version: 1.",
|
||||
"---",
|
||||
"",
|
||||
].join("\n"));
|
||||
|
||||
expect(parsed.frontmatter.version).toBe("1.");
|
||||
});
|
||||
});
|
||||
@@ -42,7 +42,7 @@ export function parseFrontmatterMarkdown(raw: string): MarkdownDoc {
|
||||
return { frontmatter: {}, body: normalized.trim(), hasFrontmatter: false };
|
||||
}
|
||||
|
||||
const frontmatterRaw = normalized.slice(4, closing).trim();
|
||||
const frontmatterRaw = normalized.slice(4, closing);
|
||||
const body = normalized.slice(closing + 5).trim();
|
||||
return {
|
||||
frontmatter: parseYamlFrontmatter(frontmatterRaw),
|
||||
@@ -53,8 +53,9 @@ export function parseFrontmatterMarkdown(raw: string): MarkdownDoc {
|
||||
|
||||
function parseYamlFrontmatter(raw: string): Record<string, unknown> {
|
||||
const prepared = prepareYamlLines(raw);
|
||||
if (prepared.length === 0) return {};
|
||||
const parsed = parseYamlBlock(prepared, 0, prepared[0]!.indent);
|
||||
const firstContentIndex = prepared.findIndex((line) => !line.isBlank && !line.isComment);
|
||||
if (firstContentIndex < 0) return {};
|
||||
const parsed = parseYamlBlock(prepared, firstContentIndex, prepared[firstContentIndex]!.indent);
|
||||
return isPlainRecord(parsed.value) ? parsed.value : {};
|
||||
}
|
||||
|
||||
@@ -63,17 +64,22 @@ function prepareYamlLines(raw: string) {
|
||||
.split("\n")
|
||||
.map((line) => ({
|
||||
indent: line.match(/^ */)?.[0].length ?? 0,
|
||||
raw: line,
|
||||
content: line.trim(),
|
||||
}))
|
||||
.filter((line) => line.content.length > 0 && !line.content.startsWith("#"));
|
||||
isBlank: line.trim().length === 0,
|
||||
isComment: line.trim().startsWith("#"),
|
||||
}));
|
||||
}
|
||||
|
||||
function parseYamlBlock(
|
||||
lines: Array<{ indent: number; content: string }>,
|
||||
lines: Array<{ indent: number; raw: string; content: string; isBlank: boolean; isComment: boolean }>,
|
||||
startIndex: number,
|
||||
indentLevel: number,
|
||||
): { value: unknown; nextIndex: number } {
|
||||
let index = startIndex;
|
||||
while (index < lines.length && (lines[index]!.isBlank || lines[index]!.isComment)) {
|
||||
index += 1;
|
||||
}
|
||||
if (index >= lines.length || lines[index]!.indent < indentLevel) {
|
||||
return { value: {}, nextIndex: index };
|
||||
}
|
||||
@@ -83,6 +89,10 @@ function parseYamlBlock(
|
||||
const values: unknown[] = [];
|
||||
while (index < lines.length) {
|
||||
const line = lines[index]!;
|
||||
if (line.isBlank || line.isComment) {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (line.indent < indentLevel) break;
|
||||
if (line.indent !== indentLevel || !line.content.startsWith("-")) break;
|
||||
|
||||
@@ -95,6 +105,36 @@ function parseYamlBlock(
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isYamlBlockScalarIndicator(remainder)) {
|
||||
const block = parseYamlBlockScalar(lines, index, indentLevel, remainder);
|
||||
values.push(block.value);
|
||||
index = block.nextIndex;
|
||||
continue;
|
||||
}
|
||||
|
||||
const inlineObjectSeparator = remainder.indexOf(":");
|
||||
if (
|
||||
inlineObjectSeparator > 0
|
||||
&& !remainder.startsWith("\"")
|
||||
&& !remainder.startsWith("{")
|
||||
&& !remainder.startsWith("[")
|
||||
) {
|
||||
const key = remainder.slice(0, inlineObjectSeparator).trim();
|
||||
const rawValue = remainder.slice(inlineObjectSeparator + 1).trim();
|
||||
const nextObject: Record<string, unknown> = {
|
||||
[key]: parseYamlScalar(rawValue),
|
||||
};
|
||||
if (index < lines.length && lines[index]!.indent > indentLevel) {
|
||||
const nested = parseYamlBlock(lines, index, indentLevel + 2);
|
||||
if (isPlainRecord(nested.value)) {
|
||||
Object.assign(nextObject, nested.value);
|
||||
}
|
||||
index = nested.nextIndex;
|
||||
}
|
||||
values.push(nextObject);
|
||||
continue;
|
||||
}
|
||||
|
||||
values.push(parseYamlScalar(remainder));
|
||||
}
|
||||
return { value: values, nextIndex: index };
|
||||
@@ -103,6 +143,10 @@ function parseYamlBlock(
|
||||
const record: Record<string, unknown> = {};
|
||||
while (index < lines.length) {
|
||||
const line = lines[index]!;
|
||||
if (line.isBlank || line.isComment) {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (line.indent < indentLevel) break;
|
||||
if (line.indent !== indentLevel) {
|
||||
index += 1;
|
||||
@@ -124,12 +168,93 @@ function parseYamlBlock(
|
||||
index = nested.nextIndex;
|
||||
continue;
|
||||
}
|
||||
if (isYamlBlockScalarIndicator(remainder)) {
|
||||
const block = parseYamlBlockScalar(lines, index, indentLevel, remainder);
|
||||
record[key] = block.value;
|
||||
index = block.nextIndex;
|
||||
continue;
|
||||
}
|
||||
record[key] = parseYamlScalar(remainder);
|
||||
}
|
||||
|
||||
return { value: record, nextIndex: index };
|
||||
}
|
||||
|
||||
function isYamlBlockScalarIndicator(rawValue: string) {
|
||||
return /^[>|][+-]?$/.test(rawValue.trim());
|
||||
}
|
||||
|
||||
function parseYamlBlockScalar(
|
||||
lines: Array<{ indent: number; raw: string; content: string; isBlank: boolean; isComment: boolean }>,
|
||||
startIndex: number,
|
||||
parentIndent: number,
|
||||
indicator: string,
|
||||
): { value: string; nextIndex: number } {
|
||||
const trimmedIndicator = indicator.trim();
|
||||
const style = trimmedIndicator[0];
|
||||
const chomp = trimmedIndicator.endsWith("+")
|
||||
? "+"
|
||||
: trimmedIndicator.endsWith("-")
|
||||
? "-"
|
||||
: "";
|
||||
let index = startIndex;
|
||||
const collected: Array<{ indent: number; raw: string; isBlank: boolean }> = [];
|
||||
while (index < lines.length) {
|
||||
const line = lines[index]!;
|
||||
if (!line.isBlank && line.indent <= parentIndent) break;
|
||||
collected.push({ indent: line.indent, raw: line.raw, isBlank: line.isBlank });
|
||||
index += 1;
|
||||
}
|
||||
|
||||
const contentLines = collected.filter((line) => !line.isBlank);
|
||||
if (contentLines.length === 0) return { value: "", nextIndex: index };
|
||||
|
||||
const blockIndent = Math.min(...contentLines.map((line) => line.indent));
|
||||
const normalizedLines = collected.map((line) => (
|
||||
line.isBlank ? "" : line.raw.slice(Math.min(blockIndent, line.raw.length))
|
||||
));
|
||||
|
||||
const baseValue = style === "|"
|
||||
? normalizedLines.join("\n")
|
||||
: foldYamlBlockScalarLines(normalizedLines);
|
||||
|
||||
return {
|
||||
value: applyYamlBlockChomp(baseValue, chomp),
|
||||
nextIndex: index,
|
||||
};
|
||||
}
|
||||
|
||||
function foldYamlBlockScalarLines(lines: string[]) {
|
||||
let value = "";
|
||||
let pendingBlankLines = 0;
|
||||
for (const line of lines) {
|
||||
if (line === "") {
|
||||
pendingBlankLines += 1;
|
||||
continue;
|
||||
}
|
||||
if (value.length === 0) {
|
||||
value = `${"\n".repeat(pendingBlankLines)}${line}`;
|
||||
} else if (pendingBlankLines > 0) {
|
||||
value += `${"\n".repeat(pendingBlankLines + 1)}${line}`;
|
||||
} else {
|
||||
value += ` ${line}`;
|
||||
}
|
||||
pendingBlankLines = 0;
|
||||
}
|
||||
|
||||
if (pendingBlankLines > 0 && value.length > 0) {
|
||||
value += "\n".repeat(pendingBlankLines);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function applyYamlBlockChomp(value: string, chomp: "" | "+" | "-") {
|
||||
if (chomp === "+") return value;
|
||||
if (chomp === "-") return value.replace(/\n+$/u, "");
|
||||
if (value.length === 0) return value;
|
||||
return value.replace(/\n+$/u, "") + "\n";
|
||||
}
|
||||
|
||||
function parseYamlScalar(rawValue: string): unknown {
|
||||
const trimmed = rawValue.trim();
|
||||
if (trimmed === "") return "";
|
||||
@@ -138,7 +263,7 @@ function parseYamlScalar(rawValue: string): unknown {
|
||||
if (trimmed === "false") return false;
|
||||
if (trimmed === "[]") return [];
|
||||
if (trimmed === "{}") return {};
|
||||
if (/^-?\d+(\.\d)?\d*$/.test(trimmed)) return Number(trimmed);
|
||||
if (/^-?\d+(\.\d+)?$/.test(trimmed)) return Number(trimmed);
|
||||
if (
|
||||
trimmed.startsWith("\"") ||
|
||||
trimmed.startsWith("[") ||
|
||||
|
||||
Reference in New Issue
Block a user