1import TurndownService from "turndown";
2import { gfm } from "turndown-plugin-gfm";
3import { normalizeMarkdownMediaLinks } from "../media/markdown-media";
4import type { ContentBlock, ExtractedDocument } from "./document";
5 
6const turndownService = new TurndownService({
7  codeBlockStyle: "fenced",
8  headingStyle: "atx",
9  bulletListMarker: "-",
10});
11 
12turndownService.use(gfm);
13 
14function renderBlock(block: ContentBlock): string {
15  switch (block.type) {
16    case "paragraph":
17      return block.text.trim();
18    case "heading":
19      return `${"#".repeat(Math.min(Math.max(block.depth, 1), 6))} ${block.text.trim()}`;
20    case "list":
21      return block.items
22        .map((item, index) => (block.ordered ? `${index + 1}. ${item.trim()}` : `- ${item.trim()}`))
23        .join("\n");
24    case "quote":
25      return block.text
26        .split("\n")
27        .map((line) => `> ${line}`)
28        .join("\n");
29    case "code":
30      return `\`\`\`${block.language ?? ""}\n${block.code.trimEnd()}\n\`\`\``;
31    case "image":
32      return `![${block.alt ?? ""}](${block.url})`;
33    case "html":
34      return turndownService.turndown(block.html).trim();
35    case "markdown":
36      return block.markdown.trim();
37  }
38}
39 
40function isDefinedValue(value: unknown): boolean {
41  return value !== undefined && value !== null && value !== "";
42}
43 
44function renderFrontmatterValue(value: unknown): string {
45  if (typeof value === "string") {
46    if (value.includes("\n")) {
47      return `|-\n${value
48        .replace(/\r\n/g, "\n")
49        .split("\n")
50        .map((line) => `  ${line}`)
51        .join("\n")}`;
52    }
53    return JSON.stringify(value);
54  }
55  if (typeof value === "number" || typeof value === "boolean") {
56    return String(value);
57  }
58  return JSON.stringify(value);
59}
60 
61function renderFrontmatter(document: ExtractedDocument): string {
62  const fields = new Map<string, unknown>();
63  const preferredOrder = [
64    "title",
65    "url",
66    "requestedUrl",
67    "author",
68    "authorName",
69    "authorUsername",
70    "authorUrl",
71    "coverImage",
72    "siteName",
73    "publishedAt",
74    "summary",
75    "adapter",
76  ];
77 
78  fields.set("title", document.title);
79  fields.set("url", document.canonicalUrl ?? document.url);
80  fields.set("requestedUrl", document.requestedUrl ?? document.url);
81  fields.set("author", document.author);
82  fields.set("siteName", document.siteName);
83  fields.set("publishedAt", document.publishedAt);
84  fields.set("summary", document.summary);
85  fields.set("adapter", document.adapter);
86 
87  for (const [key, value] of Object.entries(document.metadata ?? {})) {
88    if (!fields.has(key)) {
89      fields.set(key, value);
90    }
91  }
92 
93  const orderedKeys = [
94    ...preferredOrder.filter((key) => fields.has(key)),
95    ...Array.from(fields.keys()).filter((key) => !preferredOrder.includes(key)).sort(),
96  ];
97 
98  const lines = orderedKeys
99    .map((key) => [key, fields.get(key)] as const)
100    .filter(([, value]) => isDefinedValue(value))
101    .map(([key, value]) => `${key}: ${renderFrontmatterValue(value)}`);
102 
103  if (lines.length === 0) {
104    return "";
105  }
106 
107  return `---\n${lines.join("\n")}\n---`;
108}
109 
110function cleanMarkdown(markdown: string): string {
111  return normalizeMarkdownMediaLinks(markdown.replace(/\n{3,}/g, "\n\n").trim());
112}
113 
114function normalizeComparableTitle(value: string): string {
115  return value
116    .trim()
117    .toLowerCase()
118    .replace(/^>\s*/, "")
119    .replace(/^#+\s+/, "")
120    .replace(/(?:\.{3}|…)\s*$/, "");
121}
122 
123function bodyStartsWithTitle(body: string, title: string): boolean {
124  const firstMeaningfulLine = body
125    .replace(/\r\n/g, "\n")
126    .split("\n")
127    .map((line) => line.trim())
128    .find((line) => line && !/^!?\[[^\]]*\]\([^)]+\)$/.test(line));
129 
130  if (!firstMeaningfulLine) {
131    return false;
132  }
133 
134  const comparableTitle = normalizeComparableTitle(title);
135  const comparableFirstLine = normalizeComparableTitle(firstMeaningfulLine);
136  if (!comparableTitle || !comparableFirstLine) {
137    return false;
138  }
139 
140  return (
141    comparableFirstLine === comparableTitle ||
142    comparableFirstLine.startsWith(comparableTitle) ||
143    comparableTitle.startsWith(comparableFirstLine)
144  );
145}
146 
147export function renderMarkdown(document: ExtractedDocument): string {
148  const sections: string[] = [];
149  const frontmatter = renderFrontmatter(document);
150 
151  if (frontmatter) {
152    sections.push(frontmatter);
153  }
154 
155  const body = document.content
156    .map((block) => renderBlock(block))
157    .filter(Boolean)
158    .join("\n\n");
159 
160  if (document.title && !bodyStartsWithTitle(body, document.title)) {
161    sections.push(`# ${document.title}`);
162  }
163 
164  if (body) {
165    sections.push(body);
166  }
167 
168  return cleanMarkdown(sections.join("\n\n"));
169}
170