Source from repo
URL to Markdown

Fetch any URL via Chrome CDP and convert the rendered page to clean markdown with YouTube transcript support.
jimliuGitHub jimliuSource repo Original GitHub link Publisher page
Files
Skill
n/a
Size
271.9 KB
Entrypoint
SKILL.md
Format
git-repo
Open file
scripts/lib/adapters/hn/index.ts

Syntax-highlighted preview of this file as included in the skill package.
Rendered Source
code392 linesFree
scripts/lib/adapters/hn/index.ts
1import { JSDOM } from "jsdom";
2import TurndownService from "turndown";
3import { gfm } from "turndown-plugin-gfm";
4import type { Adapter } from "../types";
5import type { ExtractedDocument } from "../../extract/document";
6import { collectMediaFromDocument } from "../../media/markdown-media";
7 
8const HN_BASE_URL = "https://news.ycombinator.com";
9 
10const turndown = new TurndownService({
11  headingStyle: "atx",
12  bulletListMarker: "-",
13  codeBlockStyle: "fenced",
14});
15 
16turndown.use(gfm);
17 
18export interface HnItem {
19  id: number;
20  type: "story" | "comment" | "job" | "poll" | "pollopt" | string;
21  by?: string;
22  time?: number;
23  text?: string;
24  title?: string;
25  url?: string;
26  score?: number;
27  descendants?: number;
28  kids?: number[];
29  parent?: number;
30  deleted?: boolean;
31  dead?: boolean;
32}
33 
34export interface HnCommentNode {
35  item: HnItem;
36  children: HnCommentNode[];
37}
38 
39interface ParsedHnThread {
40  story: HnItem;
41  comments: HnCommentNode[];
42}
43 
44function decodeHtmlText(value: string | undefined): string | undefined {
45  if (!value) {
46    return undefined;
47  }
48 
49  const dom = new JSDOM(`<!doctype html><html><body>${value}</body></html>`);
50  return dom.window.document.body.textContent?.trim() || undefined;
51}
52 
53function normalizeMarkdown(markdown: string): string {
54  return markdown
55    .replace(/\r\n/g, "\n")
56    .replace(/[ \t]+\n/g, "\n")
57    .replace(/\n{3,}/g, "\n\n")
58    .trim();
59}
60 
61function convertHnHtmlToMarkdown(html: string | undefined, baseUrl: string): string {
62  if (!html?.trim()) {
63    return "";
64  }
65 
66  const dom = new JSDOM(`<div id="__root">${html}</div>`, { url: baseUrl });
67  const root = dom.window.document.querySelector("#__root");
68  if (!root) {
69    return "";
70  }
71 
72  root.querySelectorAll("a[href]").forEach((element) => {
73    const href = element.getAttribute("href");
74    if (!href) {
75      return;
76    }
77 
78    try {
79      element.setAttribute("href", new URL(href, baseUrl).toString());
80    } catch {
81      // Ignore malformed URLs and keep the original href.
82    }
83  });
84 
85  return normalizeMarkdown(turndown.turndown(root.innerHTML));
86}
87 
88function formatIsoTimestamp(unixSeconds: number | undefined): string | undefined {
89  if (!unixSeconds || !Number.isFinite(unixSeconds)) {
90    return undefined;
91  }
92 
93  return new Date(unixSeconds * 1_000).toISOString();
94}
95 
96function formatDisplayTimestamp(unixSeconds: number | undefined): string {
97  const iso = formatIsoTimestamp(unixSeconds);
98  if (!iso) {
99    return "unknown time";
100  }
101 
102  return iso.replace("T", " ").replace(".000Z", " UTC");
103}
104 
105function indentMarkdown(markdown: string, spaces: number): string {
106  const prefix = " ".repeat(spaces);
107  return markdown
108    .split("\n")
109    .map((line) => (line ? `${prefix}${line}` : prefix))
110    .join("\n");
111}
112 
113function renderCommentHeader(item: HnItem, pageUrl: string): string {
114  const author = item.by ?? "[deleted]";
115  const time = item.id
116    ? `[${formatDisplayTimestamp(item.time)}](${pageUrl}#${item.id})`
117    : formatDisplayTimestamp(item.time);
118  return `${author} · ${time}`;
119}
120 
121function renderCommentNode(node: HnCommentNode, pageUrl: string, depth = 0): string {
122  const baseIndent = " ".repeat(depth * 4);
123  const lines = [`${baseIndent}- ${renderCommentHeader(node.item, pageUrl)}`];
124  const body = convertHnHtmlToMarkdown(node.item.text, pageUrl);
125 
126  if (body) {
127    lines.push("");
128    lines.push(indentMarkdown(body, depth * 4 + 4));
129  } else if (node.item.deleted || node.item.dead) {
130    lines.push("");
131    lines.push(`${baseIndent}    [comment unavailable]`);
132  }
133 
134  for (const child of node.children) {
135    lines.push("");
136    lines.push(renderCommentNode(child, pageUrl, depth + 1));
137  }
138 
139  return lines.join("\n");
140}
141 
142export function buildHnThreadMarkdown(
143  story: HnItem,
144  comments: HnCommentNode[],
145  pageUrl: string,
146): string {
147  const lines: string[] = [];
148  const storyUrl = story.url ? new URL(story.url, pageUrl).toString() : undefined;
149  const storyText = convertHnHtmlToMarkdown(story.text, pageUrl);
150 
151  if (storyUrl && storyUrl !== pageUrl) {
152    lines.push(`Source: [${storyUrl}](${storyUrl})`);
153  }
154  lines.push(`HN Item: [${story.id}](${pageUrl})`);
155 
156  const submittedBy = story.by ? ` by ${story.by}` : "";
157  const submittedAt = formatDisplayTimestamp(story.time);
158  lines.push(`Submitted${submittedBy} at ${submittedAt}`);
159 
160  const stats: string[] = [];
161  if (typeof story.score === "number") {
162    stats.push(`${story.score} points`);
163  }
164  if (typeof story.descendants === "number") {
165    stats.push(`${story.descendants} comments`);
166  }
167  if (stats.length > 0) {
168    lines.push(stats.join(" | "));
169  }
170 
171  if (storyText) {
172    lines.push("");
173    lines.push("## Post");
174    lines.push("");
175    lines.push(storyText);
176  }
177 
178  lines.push("");
179  lines.push("## Comments");
180  lines.push("");
181 
182  if (comments.length === 0) {
183    lines.push("No comments.");
184  } else {
185    lines.push(comments.map((comment) => renderCommentNode(comment, pageUrl)).join("\n\n"));
186  }
187 
188  return normalizeMarkdown(lines.join("\n"));
189}
190 
191export function buildHnDocument(
192  story: HnItem,
193  comments: HnCommentNode[],
194  pageUrl: string,
195): ExtractedDocument {
196  const decodedTitle = decodeHtmlText(story.title) ?? `HN Item ${story.id}`;
197 
198  return {
199    url: pageUrl,
200    canonicalUrl: pageUrl,
201    title: decodedTitle,
202    author: story.by,
203    siteName: "Hacker News",
204    publishedAt: formatIsoTimestamp(story.time),
205    adapter: "hn",
206    metadata: {
207      kind: "hn/story",
208      storyId: story.id,
209      storyUrl: story.url ? new URL(story.url, pageUrl).toString() : undefined,
210      points: story.score,
211      commentCount: story.descendants,
212    },
213    content: [
214      {
215        type: "markdown",
216        markdown: buildHnThreadMarkdown(story, comments, pageUrl),
217      },
218    ],
219  };
220}
221 
222export function parseHnItemId(url: URL): number | null {
223  if (url.hostname !== "news.ycombinator.com") {
224    return null;
225  }
226 
227  if (url.pathname !== "/item") {
228    return null;
229  }
230 
231  const value = url.searchParams.get("id");
232  if (!value || !/^\d+$/.test(value)) {
233    return null;
234  }
235 
236  return Number(value);
237}
238 
239function extractUnixSecondsFromAge(element: Element | null): number | undefined {
240  const title = element?.getAttribute("title")?.trim();
241  if (!title) {
242    return undefined;
243  }
244 
245  const match = title.match(/(\d{9,})$/);
246  return match ? Number(match[1]) : undefined;
247}
248 
249function extractScore(text: string | null | undefined): number | undefined {
250  if (!text) {
251    return undefined;
252  }
253 
254  const match = text.match(/(\d+)/);
255  return match ? Number(match[1]) : undefined;
256}
257 
258function extractCommentCount(container: ParentNode): number | undefined {
259  const anchors = Array.from(container.querySelectorAll("a"));
260  for (const anchor of anchors) {
261    const match = anchor.textContent?.trim().match(/(\d+)\s+comments?/i);
262    if (match) {
263      return Number(match[1]);
264    }
265  }
266  return undefined;
267}
268 
269function normalizeStoryUrl(storyId: number, href: string | null | undefined, pageUrl: string): string | undefined {
270  if (!href) {
271    return undefined;
272  }
273 
274  try {
275    const resolved = new URL(href, pageUrl).toString();
276    if (resolved === pageUrl || resolved === `${HN_BASE_URL}/item?id=${storyId}`) {
277      return undefined;
278    }
279    return resolved;
280  } catch {
281    return undefined;
282  }
283}
284 
285export function extractHnThreadFromHtml(html: string, pageUrl: string): ParsedHnThread | null {
286  const dom = new JSDOM(html, { url: pageUrl });
287  const { document } = dom.window;
288  const storyRow = document.querySelector("table.fatitem tr.athing.submission");
289  if (!storyRow) {
290    return null;
291  }
292 
293  const storyId = Number(storyRow.getAttribute("id"));
294  if (!Number.isFinite(storyId)) {
295    return null;
296  }
297 
298  const titleLink = storyRow.querySelector(".titleline > a");
299  const subline = document.querySelector("table.fatitem .subline");
300  const topText = document.querySelector("table.fatitem .toptext");
301 
302  const story: HnItem = {
303    id: storyId,
304    type: "story",
305    by: subline?.querySelector(".hnuser")?.textContent?.trim() || undefined,
306    time: extractUnixSecondsFromAge(subline?.querySelector(".age") ?? null),
307    title: titleLink?.innerHTML?.trim() || undefined,
308    url: normalizeStoryUrl(storyId, titleLink?.getAttribute("href"), pageUrl),
309    text: topText?.innerHTML?.trim() || undefined,
310    score: extractScore(subline?.querySelector(".score")?.textContent),
311    descendants: extractCommentCount(subline ?? document),
312  };
313 
314  const roots: HnCommentNode[] = [];
315  const stack: HnCommentNode[] = [];
316 
317  document.querySelectorAll("tr.athing.comtr").forEach((row) => {
318    const commentId = Number(row.getAttribute("id"));
319    if (!Number.isFinite(commentId)) {
320      return;
321    }
322 
323    const indentRaw = row.querySelector("td.ind")?.getAttribute("indent");
324    const depth = indentRaw && /^\d+$/.test(indentRaw) ? Number(indentRaw) : 0;
325    const comhead = row.querySelector(".comhead");
326    const item: HnItem = {
327      id: commentId,
328      type: "comment",
329      by: comhead?.querySelector(".hnuser")?.textContent?.trim() || undefined,
330      time: extractUnixSecondsFromAge(comhead?.querySelector(".age") ?? null),
331      text: row.querySelector(".comment > .commtext")?.innerHTML?.trim() || undefined,
332      deleted: row.querySelector(".comment > .commtext") === null,
333    };
334 
335    const node: HnCommentNode = {
336      item,
337      children: [],
338    };
339 
340    while (stack.length > depth) {
341      stack.pop();
342    }
343 
344    const parent = stack[stack.length - 1];
345    if (parent) {
346      parent.children.push(node);
347    } else {
348      roots.push(node);
349    }
350 
351    stack.push(node);
352  });
353 
354  return {
355    story,
356    comments: roots,
357  };
358}
359 
360export const hnAdapter: Adapter = {
361  name: "hn",
362  match(input) {
363    return parseHnItemId(input.url) !== null;
364  },
365  async process(context) {
366    const itemId = parseHnItemId(context.input.url);
367    if (!itemId) {
368      return {
369        status: "no_document",
370      };
371    }
372 
373    const pageUrl = context.input.url.toString();
374    context.log.info(`Loading ${pageUrl} with hn adapter`);
375    await context.browser.goto(pageUrl, context.timeoutMs);
376    const html = await context.browser.getHTML();
377    const thread = extractHnThreadFromHtml(html, pageUrl);
378    if (!thread) {
379      return {
380        status: "no_document",
381      };
382    }
383 
384    const document = buildHnDocument(thread.story, thread.comments, pageUrl);
385    return {
386      status: "ok",
387      document,
388      media: collectMediaFromDocument(document),
389    };
390  },
391};
392
Preparing the source view

URL to Markdown

scripts/lib/adapters/hn/index.ts
