1import { Readability } from "@mozilla/readability";
2import { Defuddle } from "defuddle/node";
3import { JSDOM, VirtualConsole } from "jsdom";
4import TurndownService from "turndown";
5import { gfm } from "turndown-plugin-gfm";
6import { collectMediaFromMarkdown } from "../media/markdown-media";
7import type { MediaAsset } from "../media/types";
8import { cleanHtml } from "./html-cleaner";
9 
10export interface HtmlConversionMetadata {
11  url: string;
12  canonicalUrl?: string;
13  siteName?: string;
14  title?: string;
15  summary?: string;
16  author?: string;
17  publishedAt?: string;
18  coverImage?: string;
19  language?: string;
20  capturedAt: string;
21}
22 
23export interface ConvertHtmlToMarkdownOptions {
24  enableRemoteMarkdownFallback?: boolean;
25  preserveBase64Images?: boolean;
26}
27 
28export interface HtmlToMarkdownResult {
29  metadata: HtmlConversionMetadata;
30  markdown: string;
31  rawHtml: string;
32  cleanedHtml: string;
33  media: MediaAsset[];
34  conversionMethod: string;
35  fallbackReason?: string;
36}
37 
38type JsonObject = Record<string, unknown>;
39 
40const MIN_CONTENT_LENGTH = 120;
41const DEFUDDLE_API_ORIGIN = "https://defuddle.md";
42const LOCAL_FALLBACK_SCORE_DELTA = 120;
43const REMOTE_FALLBACK_SCORE_DELTA = 20;
44const LOW_QUALITY_MARKERS = [
45  /Join The Conversation/i,
46  /One Community\. Many Voices/i,
47  /Read our community guidelines/i,
48  /Create a free account to share your thoughts/i,
49  /Become a Forbes Member/i,
50  /Subscribe to trusted journalism/i,
51  /\bComments\b/i,
52];
53 
54const ARTICLE_TYPES = new Set([
55  "Article",
56  "NewsArticle",
57  "BlogPosting",
58  "WebPage",
59  "ReportageNewsArticle",
60]);
61 
62const turndown = new TurndownService({
63  headingStyle: "atx",
64  bulletListMarker: "-",
65  codeBlockStyle: "fenced",
66}) as TurndownService & {
67  remove(selectors: string[]): void;
68  addRule(
69    key: string,
70    rule: {
71      filter: string | ((node: Node) => boolean);
72      replacement: (content: string) => string;
73    },
74  ): void;
75};
76 
77turndown.use(gfm);
78turndown.remove(["script", "style", "iframe", "noscript", "template", "svg", "path"]);
79turndown.addRule("collapseFigure", {
80  filter: "figure",
81  replacement(content: string) {
82    return `\n\n${content.trim()}\n\n`;
83  },
84});
85turndown.addRule("dropInvisibleAnchors", {
86  filter(node: Node) {
87    return (
88      node.nodeName === "A" &&
89      !(node as Element).textContent?.trim() &&
90      !(node as Element).querySelector("img, video, picture, source")
91    );
92  },
93  replacement() {
94    return "";
95  },
96});
97 
98function pickString(...values: unknown[]): string | undefined {
99  for (const value of values) {
100    if (typeof value !== "string") {
101      continue;
102    }
103    const trimmed = value.trim();
104    if (trimmed) {
105      return trimmed;
106    }
107  }
108  return undefined;
109}
110 
111function normalizeMarkdown(markdown: string): string {
112  return markdown
113    .replace(/\r\n/g, "\n")
114    .replace(/[ \t]+\n/g, "\n")
115    .replace(/\n{3,}/g, "\n\n")
116    .trim();
117}
118 
119function stripWrappingQuotes(value: string): string {
120  const trimmed = value.trim();
121  if (
122    (trimmed.startsWith('"') && trimmed.endsWith('"')) ||
123    (trimmed.startsWith("'") && trimmed.endsWith("'"))
124  ) {
125    return trimmed.slice(1, -1).trim();
126  }
127  return trimmed;
128}
129 
130function stripMarkdownFrontmatter(markdown: string): string {
131  return markdown.replace(/^\uFEFF?---\n[\s\S]*?\n---(?:\n|$)/, "").trim();
132}
133 
134function cleanMarkdownTitle(value: string): string | undefined {
135  const cleaned = stripWrappingQuotes(
136    value
137      .replace(/\s+#+\s*$/, "")
138      .replace(/!\[[^\]]*\]\([^)]+\)/g, "")
139      .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
140      .replace(/[*_`~]/g, "")
141      .trim(),
142  );
143 
144  return cleaned || undefined;
145}
146 
147export function extractTitleFromMarkdownDocument(markdown: string): string | undefined {
148  const normalized = markdown.replace(/\r\n/g, "\n").trim();
149  if (!normalized) {
150    return undefined;
151  }
152 
153  const frontmatterMatch = normalized.match(/^\uFEFF?---\n([\s\S]*?)\n---(?:\n|$)/);
154  if (frontmatterMatch) {
155    for (const line of frontmatterMatch[1].split("\n")) {
156      const match = line.match(/^title:\s*(.+?)\s*$/i);
157      if (!match) {
158        continue;
159      }
160 
161      const title = cleanMarkdownTitle(match[1]);
162      if (title) {
163        return title;
164      }
165    }
166  }
167 
168  const body = stripMarkdownFrontmatter(normalized);
169  const headingMatch = body.match(/^#{1,6}\s+(.+)$/m);
170  if (!headingMatch) {
171    return undefined;
172  }
173 
174  return cleanMarkdownTitle(headingMatch[1]);
175}
176 
177function trimKnownBoilerplate(markdown: string): string {
178  const normalized = normalizeMarkdown(markdown);
179  const lines = normalized.split("\n");
180 
181  while (lines.length > 0) {
182    const lastLine = lines[lines.length - 1]?.trim();
183    if (!lastLine) {
184      lines.pop();
185      continue;
186    }
187 
188    if (/^继续滑动看下一个$/.test(lastLine) || /^轻触阅读原文$/.test(lastLine)) {
189      lines.pop();
190      continue;
191    }
192 
193    break;
194  }
195 
196  return normalizeMarkdown(lines.join("\n"));
197}
198 
199function buildDefuddleApiUrl(targetUrl: string): string {
200  return `${DEFUDDLE_API_ORIGIN}/${encodeURIComponent(targetUrl)}`;
201}
202 
203async function fetchDefuddleApiMarkdown(
204  targetUrl: string,
205): Promise<{ markdown: string; title?: string }> {
206  const response = await fetch(buildDefuddleApiUrl(targetUrl), {
207    headers: {
208      accept: "text/markdown,text/plain;q=0.9,*/*;q=0.1",
209    },
210    redirect: "follow",
211  });
212 
213  if (!response.ok) {
214    throw new Error(`defuddle.md returned ${response.status} ${response.statusText}`);
215  }
216 
217  const rawMarkdown = (await response.text()).replace(/\r\n/g, "\n").trim();
218  if (!rawMarkdown) {
219    throw new Error("defuddle.md returned empty markdown");
220  }
221 
222  const title = extractTitleFromMarkdownDocument(rawMarkdown);
223  const markdown = trimKnownBoilerplate(stripMarkdownFrontmatter(rawMarkdown));
224  if (!markdown) {
225    throw new Error("defuddle.md returned empty markdown");
226  }
227 
228  return {
229    markdown,
230    title,
231  };
232}
233 
234function sanitizeHtmlFragment(html: string): string {
235  const dom = new JSDOM(`<div id="__root">${html}</div>`);
236  const root = dom.window.document.querySelector("#__root");
237  if (!root) {
238    return html;
239  }
240 
241  for (const selector of ["script", "style", "iframe", "noscript", "template", "svg", "path"]) {
242    root.querySelectorAll(selector).forEach((element) => element.remove());
243  }
244 
245  return root.innerHTML;
246}
247 
248function extractTextFromHtml(html: string): string {
249  const dom = new JSDOM(`<!doctype html><html><body>${html}</body></html>`);
250  const { document } = dom.window;
251  for (const selector of ["script", "style", "noscript", "template", "iframe", "svg", "path"]) {
252    document.querySelectorAll(selector).forEach((element) => element.remove());
253  }
254  return document.body?.textContent?.replace(/\s+/g, " ").trim() ?? "";
255}
256 
257function getMetaContent(document: Document, names: string[]): string | undefined {
258  for (const name of names) {
259    const element =
260      document.querySelector(`meta[name="${name}"]`) ??
261      document.querySelector(`meta[property="${name}"]`);
262    const content = element?.getAttribute("content")?.trim();
263    if (content) {
264      return content;
265    }
266  }
267  return undefined;
268}
269 
270function normalizeLanguageTag(value: string | null | undefined): string | undefined {
271  if (!value) {
272    return undefined;
273  }
274 
275  const trimmed = value.trim();
276  if (!trimmed) {
277    return undefined;
278  }
279 
280  const primary = trimmed.split(/[,\s;]/, 1)[0]?.trim();
281  if (!primary) {
282    return undefined;
283  }
284 
285  return primary.replace(/_/g, "-");
286}
287 
288function flattenJsonLdItems(data: unknown): JsonObject[] {
289  if (!data || typeof data !== "object") {
290    return [];
291  }
292 
293  if (Array.isArray(data)) {
294    return data.flatMap(flattenJsonLdItems);
295  }
296 
297  const item = data as JsonObject;
298  if (Array.isArray(item["@graph"])) {
299    return (item["@graph"] as unknown[]).flatMap(flattenJsonLdItems);
300  }
301 
302  return [item];
303}
304 
305function parseJsonLdScripts(document: Document): JsonObject[] {
306  const results: JsonObject[] = [];
307  document.querySelectorAll("script[type='application/ld+json']").forEach((script) => {
308    try {
309      const data = JSON.parse(script.textContent ?? "");
310      results.push(...flattenJsonLdItems(data));
311    } catch {
312      // Ignore malformed json-ld blocks.
313    }
314  });
315  return results;
316}
317 
318function extractAuthorFromJsonLd(authorData: unknown): string | undefined {
319  if (typeof authorData === "string") {
320    return authorData.trim() || undefined;
321  }
322 
323  if (!authorData || typeof authorData !== "object") {
324    return undefined;
325  }
326 
327  if (Array.isArray(authorData)) {
328    return authorData
329      .map((author) => extractAuthorFromJsonLd(author))
330      .filter((value): value is string => Boolean(value))
331      .join(", ") || undefined;
332  }
333 
334  const author = authorData as JsonObject;
335  return pickString(author.name);
336}
337 
338function extractPrimaryJsonLdMeta(document: Document): Partial<HtmlConversionMetadata> {
339  for (const item of parseJsonLdScripts(document)) {
340    const type = Array.isArray(item["@type"]) ? item["@type"][0] : item["@type"];
341    if (typeof type !== "string" || !ARTICLE_TYPES.has(type)) {
342      continue;
343    }
344 
345    return {
346      title: pickString(item.headline, item.name),
347      summary: pickString(item.description),
348      author: extractAuthorFromJsonLd(item.author),
349      publishedAt: pickString(item.datePublished, item.dateCreated),
350      coverImage: pickString(
351        item.image,
352        (item.image as JsonObject | undefined)?.url,
353        Array.isArray(item.image) ? item.image[0] : undefined,
354      ),
355    };
356  }
357 
358  return {};
359}
360 
361function extractPageMetadata(
362  html: string,
363  url: string,
364  capturedAt: string,
365): HtmlConversionMetadata {
366  const dom = new JSDOM(html, { url });
367  const { document } = dom.window;
368  const jsonLd = extractPrimaryJsonLdMeta(document);
369 
370  return {
371    url,
372    canonicalUrl:
373      document.querySelector('link[rel="canonical"]')?.getAttribute("href")?.trim() ??
374      getMetaContent(document, ["og:url"]),
375    siteName: pickString(
376      getMetaContent(document, ["og:site_name"]),
377      document.querySelector('meta[name="application-name"]')?.getAttribute("content"),
378    ),
379    title: pickString(
380      getMetaContent(document, ["og:title", "twitter:title"]),
381      jsonLd.title,
382      document.querySelector("h1")?.textContent,
383      document.title,
384    ),
385    summary: pickString(
386      getMetaContent(document, ["description", "og:description", "twitter:description"]),
387      jsonLd.summary,
388    ),
389    author: pickString(
390      getMetaContent(document, ["author", "article:author", "twitter:creator"]),
391      jsonLd.author,
392    ),
393    publishedAt: pickString(
394      document.querySelector("time[datetime]")?.getAttribute("datetime"),
395      getMetaContent(document, ["article:published_time", "datePublished", "publishdate", "date"]),
396      jsonLd.publishedAt,
397    ),
398    coverImage: pickString(
399      getMetaContent(document, ["og:image", "twitter:image", "twitter:image:src"]),
400      jsonLd.coverImage,
401    ),
402    language: pickString(
403      normalizeLanguageTag(document.documentElement.getAttribute("lang")),
404      normalizeLanguageTag(
405        pickString(
406          getMetaContent(document, ["language", "content-language", "og:locale"]),
407          document.querySelector("meta[http-equiv='content-language']")?.getAttribute("content"),
408        ),
409      ),
410    ),
411    capturedAt,
412  };
413}
414 
415function isMarkdownUsable(markdown: string, html: string): boolean {
416  const normalized = normalizeMarkdown(markdown);
417  if (!normalized) {
418    return false;
419  }
420 
421  const htmlTextLength = extractTextFromHtml(html).length;
422  if (htmlTextLength < MIN_CONTENT_LENGTH) {
423    return true;
424  }
425 
426  if (normalized.length >= 80) {
427    return true;
428  }
429 
430  return normalized.length >= Math.min(200, Math.floor(htmlTextLength * 0.2));
431}
432 
433function countMarkerHits(markdown: string, markers: RegExp[]): number {
434  let hits = 0;
435  for (const marker of markers) {
436    if (marker.test(markdown)) {
437      hits += 1;
438    }
439  }
440  return hits;
441}
442 
443function countUsefulParagraphs(markdown: string): number {
444  const paragraphs = normalizeMarkdown(markdown).split(/\n{2,}/);
445  let count = 0;
446 
447  for (const paragraph of paragraphs) {
448    const trimmed = paragraph.trim();
449    if (!trimmed) {
450      continue;
451    }
452    if (/^!?\[[^\]]*\]\([^)]+\)$/.test(trimmed)) {
453      continue;
454    }
455    if (/^#{1,6}\s+/.test(trimmed)) {
456      continue;
457    }
458    if ((trimmed.match(/\b[\p{L}\p{N}']+\b/gu) || []).length < 8) {
459      continue;
460    }
461    count += 1;
462  }
463 
464  return count;
465}
466 
467function scoreMarkdownQuality(markdown: string): number {
468  const normalized = normalizeMarkdown(markdown);
469  const wordCount = (normalized.match(/\b[\p{L}\p{N}']+\b/gu) || []).length;
470  const usefulParagraphs = countUsefulParagraphs(normalized);
471  const headingCount = (normalized.match(/^#{1,6}\s+/gm) || []).length;
472  const markerHits = countMarkerHits(normalized, LOW_QUALITY_MARKERS);
473  return Math.min(wordCount, 4000) + usefulParagraphs * 40 + headingCount * 10 - markerHits * 180;
474}
475 
476function shouldCompareWithFallback(markdown: string): boolean {
477  const normalized = normalizeMarkdown(markdown);
478  return countMarkerHits(normalized, LOW_QUALITY_MARKERS) > 0 || countUsefulParagraphs(normalized) < 6;
479}
480 
481function hasMeaningfulMarkdownStructure(markdown: string): boolean {
482  const normalized = normalizeMarkdown(markdown);
483  if (!normalized) {
484    return false;
485  }
486 
487  return (
488    countUsefulParagraphs(normalized) > 0 ||
489    /^#{1,6}\s+/m.test(normalized) ||
490    /^[-*]\s+/m.test(normalized) ||
491    /^\d+\.\s+/m.test(normalized) ||
492    /!\[[^\]]*\]\([^)]+\)/.test(normalized)
493  );
494}
495 
496function shouldTryRemoteMarkdownFallback(
497  markdown: string,
498  html: string,
499  options: ConvertHtmlToMarkdownOptions,
500): boolean {
501  if (!options.enableRemoteMarkdownFallback) {
502    return false;
503  }
504 
505  return !isMarkdownUsable(markdown, html) || shouldCompareWithFallback(markdown);
506}
507 
508function shouldPreferRemoteMarkdown(
509  current: HtmlToMarkdownResult,
510  remote: HtmlToMarkdownResult,
511  html: string,
512): boolean {
513  if (!isMarkdownUsable(current.markdown, html)) {
514    return true;
515  }
516 
517  if (!hasMeaningfulMarkdownStructure(current.markdown) && hasMeaningfulMarkdownStructure(remote.markdown)) {
518    return true;
519  }
520 
521  return scoreMarkdownQuality(remote.markdown) > scoreMarkdownQuality(current.markdown) + REMOTE_FALLBACK_SCORE_DELTA;
522}
523 
524function buildRemoteFallbackReason(current: HtmlToMarkdownResult, html: string): string {
525  if (!isMarkdownUsable(current.markdown, html)) {
526    return current.fallbackReason
527      ? `Used defuddle.md markdown fallback after local extraction failed: ${current.fallbackReason}`
528      : "Used defuddle.md markdown fallback after local extraction returned empty or incomplete markdown";
529  }
530 
531  return "defuddle.md produced higher-quality markdown than local extraction";
532}
533 
534async function tryDefuddleConversion(
535  html: string,
536  url: string,
537  baseMetadata: HtmlConversionMetadata,
538): Promise<{ ok: true; result: HtmlToMarkdownResult } | { ok: false; reason: string }> {
539  try {
540    const virtualConsole = new VirtualConsole();
541    virtualConsole.on("jsdomError", (error: Error & { type?: string }) => {
542      if (error.type === "css parsing" || /Could not parse CSS stylesheet/i.test(error.message)) {
543        return;
544      }
545    });
546 
547    const dom = new JSDOM(html, { url, virtualConsole });
548    const result = await Defuddle(dom, url, { markdown: true });
549    const markdown = trimKnownBoilerplate(result.content || "");
550 
551    if (!isMarkdownUsable(markdown, html)) {
552      return { ok: false, reason: "Defuddle returned empty or incomplete markdown" };
553    }
554 
555    const metadata: HtmlConversionMetadata = {
556      ...baseMetadata,
557      title: pickString(result.title, baseMetadata.title),
558      summary: pickString(result.description, baseMetadata.summary),
559      author: pickString(result.author, baseMetadata.author),
560      publishedAt: pickString(result.published, baseMetadata.publishedAt),
561      coverImage: pickString(result.image, baseMetadata.coverImage),
562      language: pickString(result.language, baseMetadata.language),
563    };
564 
565    return {
566      ok: true,
567      result: {
568        metadata,
569        markdown,
570        rawHtml: html,
571        cleanedHtml: html,
572        media: collectMediaFromMarkdown(markdown).concat(
573          metadata.coverImage
574            ? [{ url: metadata.coverImage, kind: "image", role: "cover" as const }]
575            : [],
576        ),
577        conversionMethod: "defuddle",
578      },
579    };
580  } catch (error) {
581    return {
582      ok: false,
583      reason: error instanceof Error ? error.message : String(error),
584    };
585  }
586}
587 
588async function tryDefuddleApiConversion(
589  html: string,
590  url: string,
591  baseMetadata: HtmlConversionMetadata,
592): Promise<{ ok: true; result: HtmlToMarkdownResult } | { ok: false; reason: string }> {
593  try {
594    const result = await fetchDefuddleApiMarkdown(url);
595    const markdown = result.markdown;
596 
597    if (!isMarkdownUsable(markdown, html) && scoreMarkdownQuality(markdown) < 80) {
598      return { ok: false, reason: "defuddle.md returned empty or incomplete markdown" };
599    }
600 
601    const metadata: HtmlConversionMetadata = {
602      ...baseMetadata,
603      title: pickString(result.title, baseMetadata.title),
604    };
605 
606    return {
607      ok: true,
608      result: {
609        metadata,
610        markdown,
611        rawHtml: html,
612        cleanedHtml: html,
613        media: collectMediaFromMarkdown(markdown).concat(
614          metadata.coverImage
615            ? [{ url: metadata.coverImage, kind: "image", role: "cover" as const }]
616            : [],
617        ),
618        conversionMethod: "defuddle-api",
619      },
620    };
621  } catch (error) {
622    return {
623      ok: false,
624      reason: error instanceof Error ? error.message : String(error),
625    };
626  }
627}
628 
629function convertHtmlFragmentToMarkdown(html: string): string {
630  if (!html.trim()) {
631    return "";
632  }
633 
634  try {
635    return turndown.turndown(sanitizeHtmlFragment(html));
636  } catch {
637    return "";
638  }
639}
640 
641function fallbackPlainText(html: string): string {
642  return trimKnownBoilerplate(extractTextFromHtml(html));
643}
644 
645function convertWithReadability(
646  rawHtml: string,
647  cleanedHtml: string,
648  url: string,
649  baseMetadata: HtmlConversionMetadata,
650): HtmlToMarkdownResult {
651  const dom = new JSDOM(cleanedHtml, { url });
652  const document = dom.window.document;
653  const article = new Readability(document).parse();
654 
655  const contentHtml =
656    article?.content?.trim() ??
657    document.querySelector("main")?.innerHTML?.trim() ??
658    document.body?.innerHTML?.trim() ??
659    "";
660 
661  let markdown = contentHtml ? convertHtmlFragmentToMarkdown(contentHtml) : "";
662  if (!markdown) {
663    markdown = fallbackPlainText(cleanedHtml);
664  }
665 
666  const metadata: HtmlConversionMetadata = {
667    ...baseMetadata,
668    title: pickString(article?.title, baseMetadata.title),
669    summary: pickString(article?.excerpt, baseMetadata.summary),
670    author: pickString(article?.byline, baseMetadata.author),
671  };
672 
673  const media = collectMediaFromMarkdown(markdown);
674  if (metadata.coverImage) {
675    media.unshift({
676      url: metadata.coverImage,
677      kind: "image",
678      role: "cover",
679    });
680  }
681 
682  return {
683    metadata,
684    markdown: trimKnownBoilerplate(markdown),
685    rawHtml,
686    cleanedHtml,
687    media,
688    conversionMethod: article?.content ? "legacy:readability" : "legacy:body",
689  };
690}
691 
692export async function convertHtmlToMarkdown(
693  html: string,
694  url: string,
695  options: ConvertHtmlToMarkdownOptions = {},
696): Promise<HtmlToMarkdownResult> {
697  const capturedAt = new Date().toISOString();
698  const baseMetadata = extractPageMetadata(html, url, capturedAt);
699 
700  let cleanedHtml = html;
701  try {
702    cleanedHtml = cleanHtml(html, url, {
703      removeBase64Images: !options.preserveBase64Images,
704    });
705  } catch {
706    cleanedHtml = html;
707  }
708 
709  let selectedResult: HtmlToMarkdownResult;
710  const defuddleResult = await tryDefuddleConversion(cleanedHtml, url, baseMetadata);
711  if (defuddleResult.ok) {
712    if (shouldCompareWithFallback(defuddleResult.result.markdown)) {
713      const fallbackResult = convertWithReadability(html, cleanedHtml, url, baseMetadata);
714      if (
715        scoreMarkdownQuality(fallbackResult.markdown) >
716        scoreMarkdownQuality(defuddleResult.result.markdown) + LOCAL_FALLBACK_SCORE_DELTA
717      ) {
718        selectedResult = {
719          ...fallbackResult,
720          fallbackReason: "Readability/Turndown produced higher-quality markdown than Defuddle",
721        };
722      } else {
723        selectedResult = {
724          ...defuddleResult.result,
725          rawHtml: html,
726          cleanedHtml,
727        };
728      }
729    } else {
730      selectedResult = {
731        ...defuddleResult.result,
732        rawHtml: html,
733        cleanedHtml,
734      };
735    }
736  } else {
737    selectedResult = {
738      ...convertWithReadability(html, cleanedHtml, url, baseMetadata),
739      fallbackReason: defuddleResult.reason,
740    };
741  }
742 
743  if (!shouldTryRemoteMarkdownFallback(selectedResult.markdown, cleanedHtml, options)) {
744    return selectedResult;
745  }
746 
747  const remoteDefuddleResult = await tryDefuddleApiConversion(cleanedHtml, url, baseMetadata);
748  if (!remoteDefuddleResult.ok || !shouldPreferRemoteMarkdown(selectedResult, remoteDefuddleResult.result, cleanedHtml)) {
749    return selectedResult;
750  }
751 
752  return {
753    ...remoteDefuddleResult.result,
754    rawHtml: html,
755    cleanedHtml,
756    fallbackReason: buildRemoteFallbackReason(selectedResult, cleanedHtml),
757  };
758}
759