5f16efb3d0
## Thinking Path > - Paperclip is the open source control plane teams use to manage AI agents for work. > - Company skills are imported from `SKILL.md` files and rely on YAML frontmatter to describe what each skill does. > - Multi-line descriptions commonly use YAML block scalars (`>` and `|`), but the broken parser path behind #4989 reduced those descriptions to a literal `>` or `|`. > - The earliest contributor fix for that bug was PR #5046, so this branch keeps that PR as the canonical merge target instead of replacing it. > - Follow-up work from #5071 and #8258 was then transplanted onto this earlier branch so the final PR preserves contributor credit while still shipping the strongest complete fix. > - The resulting change fixes block-scalar parsing in the shared frontmatter path, aligns server company-skill imports with that shared parser, and prevents already-stale stored markers from rendering as junk in the UI. ## Linked Issues or Issue Description - Fixes #4989 - Refs #2863 - Refs #788 - Related superseded PRs: #5071, #8258 ## What Changed - Kept the original PR #5046 server-side company-skill fix and regression coverage as the base branch history. - Added the missing YAML chomping and indicator hardening explored further in #5071. - Moved frontmatter parsing to the shared parser path so `packages/shared`, `packages/skills-catalog`, and server company-skill imports stay aligned. - Added UI summary sanitization and fallback handling so stale stored `>` / `|` values no longer render as visible junk in company-skill cards. - Added regression coverage for shared frontmatter parsing, skills-catalog parsing, company-skill imports, and stale-summary fallback behavior. ## Verification - Passed locally: `pnpm exec vitest run packages/shared/src/frontmatter.test.ts packages/skills-catalog/src/frontmatter.test.ts server/src/__tests__/company-skills.test.ts ui/src/lib/company-skill-summary.test.ts` - Passed locally: `pnpm --filter @paperclipai/shared typecheck` - Passed locally: `pnpm --filter @paperclipai/skills-catalog typecheck` - Not fully runnable in this worktree: `pnpm --filter @paperclipai/server typecheck` currently fails in `packages/plugins/sdk` before reaching server code because local workspace `node_modules` type deps are missing (`TS2688` for `node` / `react`). - GitHub Actions / PR checks are rerunning on PR #5046 head `005290b7557725abf748d00f36dd24ea0d919aba`. ## Risks - Medium-low risk: the fix now touches shared parser code, server company-skill imports, and UI fallback display rather than only the server import path. - The parser is still intentionally narrower than a full YAML implementation; this change focuses on block-scalar correctness and the stale-description rendering path relevant to #4989 / #2863. - This branch intentionally supersedes narrower overlapping work from #5071 and duplicate work from #8258 once the survivor PR is green. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex / GPT-5-based coding agent with local shell and code-editing tools enabled. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] If this change affects the UI, I have included before/after screenshots - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Devin Foley <devin@paperclip.ing>
280 lines
8.3 KiB
TypeScript
280 lines
8.3 KiB
TypeScript
export interface MarkdownDoc {
|
|
frontmatter: Record<string, unknown>;
|
|
body: string;
|
|
hasFrontmatter: boolean;
|
|
}
|
|
|
|
export function isPlainRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
}
|
|
|
|
export function asString(value: unknown): string | null {
|
|
if (typeof value !== "string") return null;
|
|
const trimmed = value.trim();
|
|
return trimmed.length > 0 ? trimmed : null;
|
|
}
|
|
|
|
export function asBoolean(value: unknown): boolean | null {
|
|
return typeof value === "boolean" ? value : null;
|
|
}
|
|
|
|
export function asStringArray(value: unknown): string[] | null {
|
|
if (value === undefined) return [];
|
|
if (!Array.isArray(value)) return null;
|
|
|
|
const out: string[] = [];
|
|
for (const item of value) {
|
|
const text = asString(item);
|
|
if (!text) return null;
|
|
out.push(text);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
export function parseFrontmatterMarkdown(raw: string): MarkdownDoc {
|
|
const normalized = raw.replace(/\r\n/g, "\n");
|
|
if (!normalized.startsWith("---\n")) {
|
|
return { frontmatter: {}, body: normalized.trim(), hasFrontmatter: false };
|
|
}
|
|
|
|
const closing = normalized.indexOf("\n---\n", 4);
|
|
if (closing < 0) {
|
|
return { frontmatter: {}, body: normalized.trim(), hasFrontmatter: false };
|
|
}
|
|
|
|
const frontmatterRaw = normalized.slice(4, closing);
|
|
const body = normalized.slice(closing + 5).trim();
|
|
return {
|
|
frontmatter: parseYamlFrontmatter(frontmatterRaw),
|
|
body,
|
|
hasFrontmatter: true,
|
|
};
|
|
}
|
|
|
|
function parseYamlFrontmatter(raw: string): Record<string, unknown> {
|
|
const prepared = prepareYamlLines(raw);
|
|
const firstContentIndex = prepared.findIndex((line) => !line.isBlank && !line.isComment);
|
|
if (firstContentIndex < 0) return {};
|
|
const parsed = parseYamlBlock(prepared, firstContentIndex, prepared[firstContentIndex]!.indent);
|
|
return isPlainRecord(parsed.value) ? parsed.value : {};
|
|
}
|
|
|
|
function prepareYamlLines(raw: string) {
|
|
return raw
|
|
.split("\n")
|
|
.map((line) => ({
|
|
indent: line.match(/^ */)?.[0].length ?? 0,
|
|
raw: line,
|
|
content: line.trim(),
|
|
isBlank: line.trim().length === 0,
|
|
isComment: line.trim().startsWith("#"),
|
|
}));
|
|
}
|
|
|
|
function parseYamlBlock(
|
|
lines: Array<{ indent: number; raw: string; content: string; isBlank: boolean; isComment: boolean }>,
|
|
startIndex: number,
|
|
indentLevel: number,
|
|
): { value: unknown; nextIndex: number } {
|
|
let index = startIndex;
|
|
while (index < lines.length && (lines[index]!.isBlank || lines[index]!.isComment)) {
|
|
index += 1;
|
|
}
|
|
if (index >= lines.length || lines[index]!.indent < indentLevel) {
|
|
return { value: {}, nextIndex: index };
|
|
}
|
|
|
|
const isArray = lines[index]!.indent === indentLevel && lines[index]!.content.startsWith("-");
|
|
if (isArray) {
|
|
const values: unknown[] = [];
|
|
while (index < lines.length) {
|
|
const line = lines[index]!;
|
|
if (line.isBlank || line.isComment) {
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (line.indent < indentLevel) break;
|
|
if (line.indent !== indentLevel || !line.content.startsWith("-")) break;
|
|
|
|
const remainder = line.content.slice(1).trim();
|
|
index += 1;
|
|
if (!remainder) {
|
|
const nested = parseYamlBlock(lines, index, indentLevel + 2);
|
|
values.push(nested.value);
|
|
index = nested.nextIndex;
|
|
continue;
|
|
}
|
|
|
|
if (isYamlBlockScalarIndicator(remainder)) {
|
|
const block = parseYamlBlockScalar(lines, index, indentLevel, remainder);
|
|
values.push(block.value);
|
|
index = block.nextIndex;
|
|
continue;
|
|
}
|
|
|
|
const inlineObjectSeparator = remainder.indexOf(":");
|
|
if (
|
|
inlineObjectSeparator > 0
|
|
&& !remainder.startsWith("\"")
|
|
&& !remainder.startsWith("{")
|
|
&& !remainder.startsWith("[")
|
|
) {
|
|
const key = remainder.slice(0, inlineObjectSeparator).trim();
|
|
const rawValue = remainder.slice(inlineObjectSeparator + 1).trim();
|
|
const nextObject: Record<string, unknown> = {
|
|
[key]: parseYamlScalar(rawValue),
|
|
};
|
|
if (index < lines.length && lines[index]!.indent > indentLevel) {
|
|
const nested = parseYamlBlock(lines, index, indentLevel + 2);
|
|
if (isPlainRecord(nested.value)) {
|
|
Object.assign(nextObject, nested.value);
|
|
}
|
|
index = nested.nextIndex;
|
|
}
|
|
values.push(nextObject);
|
|
continue;
|
|
}
|
|
|
|
values.push(parseYamlScalar(remainder));
|
|
}
|
|
return { value: values, nextIndex: index };
|
|
}
|
|
|
|
const record: Record<string, unknown> = {};
|
|
while (index < lines.length) {
|
|
const line = lines[index]!;
|
|
if (line.isBlank || line.isComment) {
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (line.indent < indentLevel) break;
|
|
if (line.indent !== indentLevel) {
|
|
index += 1;
|
|
continue;
|
|
}
|
|
|
|
const separatorIndex = line.content.indexOf(":");
|
|
if (separatorIndex <= 0) {
|
|
index += 1;
|
|
continue;
|
|
}
|
|
|
|
const key = line.content.slice(0, separatorIndex).trim();
|
|
const remainder = line.content.slice(separatorIndex + 1).trim();
|
|
index += 1;
|
|
if (!remainder) {
|
|
const nested = parseYamlBlock(lines, index, indentLevel + 2);
|
|
record[key] = nested.value;
|
|
index = nested.nextIndex;
|
|
continue;
|
|
}
|
|
if (isYamlBlockScalarIndicator(remainder)) {
|
|
const block = parseYamlBlockScalar(lines, index, indentLevel, remainder);
|
|
record[key] = block.value;
|
|
index = block.nextIndex;
|
|
continue;
|
|
}
|
|
record[key] = parseYamlScalar(remainder);
|
|
}
|
|
|
|
return { value: record, nextIndex: index };
|
|
}
|
|
|
|
function isYamlBlockScalarIndicator(rawValue: string) {
|
|
return /^[>|][+-]?$/.test(rawValue.trim());
|
|
}
|
|
|
|
function parseYamlBlockScalar(
|
|
lines: Array<{ indent: number; raw: string; content: string; isBlank: boolean; isComment: boolean }>,
|
|
startIndex: number,
|
|
parentIndent: number,
|
|
indicator: string,
|
|
): { value: string; nextIndex: number } {
|
|
const trimmedIndicator = indicator.trim();
|
|
const style = trimmedIndicator[0];
|
|
const chomp = trimmedIndicator.endsWith("+")
|
|
? "+"
|
|
: trimmedIndicator.endsWith("-")
|
|
? "-"
|
|
: "";
|
|
let index = startIndex;
|
|
const collected: Array<{ indent: number; raw: string; isBlank: boolean }> = [];
|
|
while (index < lines.length) {
|
|
const line = lines[index]!;
|
|
if (!line.isBlank && line.indent <= parentIndent) break;
|
|
collected.push({ indent: line.indent, raw: line.raw, isBlank: line.isBlank });
|
|
index += 1;
|
|
}
|
|
|
|
const contentLines = collected.filter((line) => !line.isBlank);
|
|
if (contentLines.length === 0) return { value: "", nextIndex: index };
|
|
|
|
const blockIndent = Math.min(...contentLines.map((line) => line.indent));
|
|
const normalizedLines = collected.map((line) => (
|
|
line.isBlank ? "" : line.raw.slice(Math.min(blockIndent, line.raw.length))
|
|
));
|
|
|
|
const baseValue = style === "|"
|
|
? normalizedLines.join("\n")
|
|
: foldYamlBlockScalarLines(normalizedLines);
|
|
|
|
return {
|
|
value: applyYamlBlockChomp(baseValue, chomp),
|
|
nextIndex: index,
|
|
};
|
|
}
|
|
|
|
function foldYamlBlockScalarLines(lines: string[]) {
|
|
let value = "";
|
|
let pendingBlankLines = 0;
|
|
for (const line of lines) {
|
|
if (line === "") {
|
|
pendingBlankLines += 1;
|
|
continue;
|
|
}
|
|
if (value.length === 0) {
|
|
value = `${"\n".repeat(pendingBlankLines)}${line}`;
|
|
} else if (pendingBlankLines > 0) {
|
|
value += `${"\n".repeat(pendingBlankLines + 1)}${line}`;
|
|
} else {
|
|
value += ` ${line}`;
|
|
}
|
|
pendingBlankLines = 0;
|
|
}
|
|
|
|
if (pendingBlankLines > 0 && value.length > 0) {
|
|
value += "\n".repeat(pendingBlankLines);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function applyYamlBlockChomp(value: string, chomp: "" | "+" | "-") {
|
|
if (chomp === "+") return value;
|
|
if (chomp === "-") return value.replace(/\n+$/u, "");
|
|
if (value.length === 0) return value;
|
|
return value.replace(/\n+$/u, "") + "\n";
|
|
}
|
|
|
|
function parseYamlScalar(rawValue: string): unknown {
|
|
const trimmed = rawValue.trim();
|
|
if (trimmed === "") return "";
|
|
if (trimmed === "null" || trimmed === "~") return null;
|
|
if (trimmed === "true") return true;
|
|
if (trimmed === "false") return false;
|
|
if (trimmed === "[]") return [];
|
|
if (trimmed === "{}") return {};
|
|
if (/^-?\d+(\.\d+)?$/.test(trimmed)) return Number(trimmed);
|
|
if (
|
|
trimmed.startsWith("\"") ||
|
|
trimmed.startsWith("[") ||
|
|
trimmed.startsWith("{")
|
|
) {
|
|
try {
|
|
return JSON.parse(trimmed);
|
|
} catch {
|
|
return trimmed;
|
|
}
|
|
}
|
|
return trimmed;
|
|
}
|