1import { Readability } from "@mozilla/readability";
2import { JSDOM } from "jsdom";
3import type { ExtractedDocument } from "./document";
4 
5function getMetaContent(document: Document, selectors: string[]): string | undefined {
6  for (const selector of selectors) {
7    const value = document.querySelector(selector)?.getAttribute("content")?.trim();
8    if (value) {
9      return value;
10    }
11  }
12  return undefined;
13}
14 
15export function extractDocumentFromHtml(input: {
16  url: string;
17  html: string;
18  adapter?: string;
19}): ExtractedDocument {
20  const dom = new JSDOM(input.html, { url: input.url });
21  const document = dom.window.document;
22 
23  const canonicalUrl =
24    document.querySelector('link[rel="canonical"]')?.getAttribute("href")?.trim() ??
25    getMetaContent(document, ['meta[property="og:url"]']);
26 
27  const siteName = getMetaContent(document, [
28    'meta[property="og:site_name"]',
29    'meta[name="application-name"]',
30  ]);
31 
32  const metadataAuthor = getMetaContent(document, [
33    'meta[name="author"]',
34    'meta[property="article:author"]',
35    'meta[name="twitter:creator"]',
36  ]);
37 
38  const publishedAt = getMetaContent(document, [
39    'meta[property="article:published_time"]',
40    'meta[name="pubdate"]',
41    'meta[name="date"]',
42    'meta[itemprop="datePublished"]',
43  ]);
44 
45  const article = new Readability(document).parse();
46  const title =
47    article?.title?.trim() ||
48    getMetaContent(document, ['meta[property="og:title"]']) ||
49    document.title.trim() ||
50    undefined;
51 
52  const summary =
53    article?.excerpt?.trim() ||
54    getMetaContent(document, [
55      'meta[name="description"]',
56      'meta[property="og:description"]',
57      'meta[name="twitter:description"]',
58    ]);
59 
60  const contentHtml =
61    article?.content?.trim() ||
62    document.querySelector("main")?.innerHTML?.trim() ||
63    document.body?.innerHTML?.trim() ||
64    "";
65 
66  const author = article?.byline?.trim() || metadataAuthor;
67 
68  return {
69    url: input.url,
70    canonicalUrl,
71    title,
72    author,
73    siteName,
74    publishedAt,
75    summary,
76    adapter: input.adapter ?? "generic",
77    metadata: {
78      language: document.documentElement.lang || undefined,
79    },
80    content: contentHtml ? [{ type: "html", html: contentHtml }] : [],
81  };
82}
83 
84