1import { JSDOM } from "jsdom";
2 
3export interface CleanHtmlOptions {
4  removeAds?: boolean;
5  removeBase64Images?: boolean;
6  onlyMainContent?: boolean;
7  includeSelectors?: string[];
8  excludeSelectors?: string[];
9}
10 
11const ALWAYS_REMOVE_SELECTORS = [
12  "script",
13  "style",
14  "noscript",
15  "link[rel='stylesheet']",
16  "[hidden]",
17  "[aria-hidden='true']",
18  "[style*='display: none']",
19  "[style*='display:none']",
20  "[style*='visibility: hidden']",
21  "[style*='visibility:hidden']",
22  "svg[aria-hidden='true']",
23  "svg.icon",
24  "svg[class*='icon']",
25  "template",
26  "meta",
27  "iframe",
28  "canvas",
29  "object",
30  "embed",
31  "form",
32  "input",
33  "select",
34  "textarea",
35  "button",
36];
37 
38const OVERLAY_SELECTORS = [
39  "[class*='modal']",
40  "[class*='popup']",
41  "[class*='overlay']",
42  "[class*='dialog']",
43  "[role='dialog']",
44  "[role='alertdialog']",
45  "[class*='cookie']",
46  "[class*='consent']",
47  "[class*='gdpr']",
48  "[class*='privacy-banner']",
49  "[class*='notification-bar']",
50  "[id*='cookie']",
51  "[id*='consent']",
52  "[id*='gdpr']",
53  "[style*='position: fixed']",
54  "[style*='position:fixed']",
55  "[style*='position: sticky']",
56  "[style*='position:sticky']",
57];
58 
59const NAVIGATION_SELECTORS = [
60  "header",
61  "footer",
62  "nav",
63  "aside",
64  ".header",
65  ".top",
66  ".navbar",
67  "#header",
68  ".footer",
69  ".bottom",
70  "#footer",
71  ".sidebar",
72  ".side",
73  ".aside",
74  "#sidebar",
75  ".modal",
76  ".popup",
77  "#modal",
78  ".overlay",
79  ".ad",
80  ".ads",
81  ".advert",
82  "#ad",
83  ".lang-selector",
84  ".language",
85  "#language-selector",
86  ".social",
87  ".social-media",
88  ".social-links",
89  "#social",
90  ".menu",
91  ".navigation",
92  "#nav",
93  ".breadcrumbs",
94  "#breadcrumbs",
95  ".share",
96  "#share",
97  ".widget",
98  "#widget",
99  ".cookie",
100  "#cookie",
101];
102 
103const FORCE_INCLUDE_SELECTORS = [
104  "#main",
105  "#content",
106  "#main-content",
107  "#article",
108  "#post",
109  "#page-content",
110  "main",
111  "article",
112  "[role='main']",
113  ".main-content",
114  ".content",
115  ".post-content",
116  ".article-content",
117  ".entry-content",
118  ".page-content",
119  ".article-body",
120  ".post-body",
121  ".story-content",
122  ".blog-content",
123];
124 
125const AD_SELECTORS = [
126  "ins.adsbygoogle",
127  ".google-ad",
128  ".adsense",
129  "[data-ad]",
130  "[data-ads]",
131  "[data-ad-slot]",
132  "[data-ad-client]",
133  ".ad-container",
134  ".ad-wrapper",
135  ".advertisement",
136  ".sponsored-content",
137  "img[width='1'][height='1']",
138  "img[src*='pixel']",
139  "img[src*='tracking']",
140  "img[src*='analytics']",
141];
142 
143function getLinkDensity(element: Element): number {
144  const text = element.textContent || "";
145  const textLength = text.trim().length;
146  if (textLength === 0) {
147    return 1;
148  }
149 
150  let linkLength = 0;
151  element.querySelectorAll("a").forEach((link) => {
152    linkLength += (link.textContent || "").trim().length;
153  });
154 
155  return linkLength / textLength;
156}
157 
158function getContentScore(element: Element): number {
159  let score = 0;
160  const text = element.textContent || "";
161  const textLength = text.trim().length;
162 
163  score += Math.min(textLength / 100, 50);
164  score += element.querySelectorAll("p").length * 3;
165  score += element.querySelectorAll("h1, h2, h3, h4, h5, h6").length * 2;
166  score += element.querySelectorAll("img").length;
167 
168  score -= element.querySelectorAll("a").length * 0.5;
169  score -= element.querySelectorAll("li").length * 0.2;
170 
171  const linkDensity = getLinkDensity(element);
172  if (linkDensity > 0.5) {
173    score -= 30;
174  } else if (linkDensity > 0.3) {
175    score -= 15;
176  }
177 
178  const className = typeof element.className === "string" ? element.className : "";
179  const classAndId = `${className} ${element.id || ""}`;
180  if (/article|content|post|body|main|entry/i.test(classAndId)) {
181    score += 25;
182  }
183  if (/comment|sidebar|footer|nav|menu|header|widget|ad/i.test(classAndId)) {
184    score -= 25;
185  }
186 
187  return score;
188}
189 
190function looksLikeNavigation(element: Element): boolean {
191  const linkDensity = getLinkDensity(element);
192  if (linkDensity > 0.5) {
193    return true;
194  }
195 
196  const listItems = element.querySelectorAll("li");
197  const links = element.querySelectorAll("a");
198  return listItems.length > 5 && links.length > listItems.length * 0.8;
199}
200 
201function removeElements(document: Document, selectors: string[]): void {
202  for (const selector of selectors) {
203    try {
204      document.querySelectorAll(selector).forEach((element) => element.remove());
205    } catch {
206      // Ignore unsupported selectors.
207    }
208  }
209}
210 
211function removeWithProtection(
212  document: Document,
213  selectorsToRemove: string[],
214  protectedSelectors: string[],
215): void {
216  for (const selector of selectorsToRemove) {
217    try {
218      document.querySelectorAll(selector).forEach((element) => {
219        const isProtected = protectedSelectors.some((protectedSelector) => {
220          try {
221            return element.matches(protectedSelector);
222          } catch {
223            return false;
224          }
225        });
226 
227        if (isProtected) {
228          return;
229        }
230 
231        const containsProtected = protectedSelectors.some((protectedSelector) => {
232          try {
233            return element.querySelector(protectedSelector) !== null;
234          } catch {
235            return false;
236          }
237        });
238 
239        if (containsProtected) {
240          return;
241        }
242 
243        element.remove();
244      });
245    } catch {
246      // Ignore unsupported selectors.
247    }
248  }
249}
250 
251function isValidContent(element: Element | null): element is Element {
252  if (!element) {
253    return false;
254  }
255  const text = element.textContent || "";
256  if (text.trim().length < 100) {
257    return false;
258  }
259  return !looksLikeNavigation(element);
260}
261 
262function findMainContent(document: Document): Element | null {
263  const main = document.querySelector("main");
264  if (isValidContent(main) && getLinkDensity(main) < 0.4) {
265    return main;
266  }
267 
268  const roleMain = document.querySelector('[role="main"]');
269  if (isValidContent(roleMain) && getLinkDensity(roleMain) < 0.4) {
270    return roleMain;
271  }
272 
273  const articles = document.querySelectorAll("article");
274  if (articles.length === 1 && isValidContent(articles[0] ?? null)) {
275    return articles[0] ?? null;
276  }
277 
278  const contentSelectors = [
279    "#content",
280    "#main-content",
281    "#main",
282    ".content",
283    ".main-content",
284    ".post-content",
285    ".article-content",
286    ".entry-content",
287    ".page-content",
288    ".article-body",
289    ".post-body",
290    ".story-content",
291    ".blog-content",
292  ];
293 
294  for (const selector of contentSelectors) {
295    try {
296      const element = document.querySelector(selector);
297      if (isValidContent(element) && getLinkDensity(element) < 0.4) {
298        return element;
299      }
300    } catch {
301      // Ignore invalid selectors.
302    }
303  }
304 
305  const candidates: Array<{ element: Element; score: number }> = [];
306  document.querySelectorAll("div, section, article").forEach((element) => {
307    const text = element.textContent || "";
308    if (text.trim().length < 200) {
309      return;
310    }
311 
312    const score = getContentScore(element);
313    if (score > 0) {
314      candidates.push({ element, score });
315    }
316  });
317 
318  candidates.sort((left, right) => right.score - left.score);
319  if ((candidates[0]?.score ?? 0) > 20) {
320    return candidates[0]?.element ?? null;
321  }
322 
323  return null;
324}
325 
326function removeBase64ImagesFromDocument(document: Document): void {
327  document.querySelectorAll("img[src^='data:']").forEach((element) => element.remove());
328 
329  document.querySelectorAll("[style*='data:image']").forEach((element) => {
330    const style = element.getAttribute("style");
331    if (!style) {
332      return;
333    }
334 
335    const cleanedStyle = style.replace(
336      /background(-image)?:\s*url\([^)]*data:image[^)]*\)[^;]*;?/gi,
337      "",
338    );
339 
340    if (cleanedStyle.trim()) {
341      element.setAttribute("style", cleanedStyle);
342    } else {
343      element.removeAttribute("style");
344    }
345  });
346 
347  document
348    .querySelectorAll("source[src^='data:'], source[srcset*='data:']")
349    .forEach((element) => element.remove());
350}
351 
352function makeAbsoluteUrl(value: string, baseUrl: string): string | null {
353  try {
354    return new URL(value, baseUrl).toString();
355  } catch {
356    return null;
357  }
358}
359 
360function convertRelativeUrls(document: Document, baseUrl: string): void {
361  document.querySelectorAll("[src]").forEach((element) => {
362    const src = element.getAttribute("src");
363    if (!src || src.startsWith("http") || src.startsWith("//") || src.startsWith("data:")) {
364      return;
365    }
366 
367    const absolute = makeAbsoluteUrl(src, baseUrl);
368    if (absolute) {
369      element.setAttribute("src", absolute);
370    }
371  });
372 
373  document.querySelectorAll("[href]").forEach((element) => {
374    const href = element.getAttribute("href");
375    if (
376      !href ||
377      href.startsWith("http") ||
378      href.startsWith("//") ||
379      href.startsWith("#") ||
380      href.startsWith("mailto:") ||
381      href.startsWith("tel:") ||
382      href.startsWith("javascript:")
383    ) {
384      return;
385    }
386 
387    const absolute = makeAbsoluteUrl(href, baseUrl);
388    if (absolute) {
389      element.setAttribute("href", absolute);
390    }
391  });
392}
393 
394function removeComments(document: Document): void {
395  const walker = document.createTreeWalker(document, document.defaultView?.NodeFilter.SHOW_COMMENT ?? 128);
396  const comments: Comment[] = [];
397  while (walker.nextNode()) {
398    comments.push(walker.currentNode as Comment);
399  }
400  comments.forEach((comment) => comment.parentNode?.removeChild(comment));
401}
402 
403export function cleanHtml(
404  html: string,
405  baseUrl: string,
406  options: CleanHtmlOptions = {},
407): string {
408  const {
409    removeAds = true,
410    removeBase64Images = true,
411    onlyMainContent = true,
412    includeSelectors,
413    excludeSelectors,
414  } = options;
415 
416  const dom = new JSDOM(html, { url: baseUrl });
417  const { document } = dom.window;
418 
419  removeElements(document, ALWAYS_REMOVE_SELECTORS);
420  removeElements(document, OVERLAY_SELECTORS);
421 
422  if (removeAds) {
423    removeElements(document, AD_SELECTORS);
424  }
425 
426  if (excludeSelectors?.length) {
427    removeElements(document, excludeSelectors);
428  }
429 
430  if (onlyMainContent) {
431    removeWithProtection(document, NAVIGATION_SELECTORS, FORCE_INCLUDE_SELECTORS);
432 
433    const mainContent = findMainContent(document);
434    if (mainContent && document.body) {
435      const clone = mainContent.cloneNode(true);
436      document.body.innerHTML = "";
437      document.body.appendChild(clone);
438    }
439  }
440 
441  if (includeSelectors?.length && document.body) {
442    const matchedElements: Element[] = [];
443    for (const selector of includeSelectors) {
444      try {
445        document.querySelectorAll(selector).forEach((element) => {
446          matchedElements.push(element.cloneNode(true) as Element);
447        });
448      } catch {
449        // Ignore invalid selectors.
450      }
451    }
452 
453    if (matchedElements.length > 0) {
454      document.body.innerHTML = "";
455      matchedElements.forEach((element) => document.body?.appendChild(element));
456    }
457  }
458 
459  if (removeBase64Images) {
460    removeBase64ImagesFromDocument(document);
461  }
462 
463  removeComments(document);
464  convertRelativeUrls(document, baseUrl);
465 
466  return document.documentElement.outerHTML || html;
467}
468