1import type { ExtractedDocument } from "../../extract/document";
2import {
3  findTweetNode,
4  findTweetNodeById,
5  formatMediaList,
6  formatTweetAuthor,
7  getTweetAuthorMetadata,
8  getTweetText,
9  getUser,
10  isRecord,
11  normalizeTitle,
12  resolveBestXVideoVariantUrl,
13  toHighResXImageUrl,
14  toXTweet,
15} from "./shared";
16import type { JsonObject } from "./types";
17 
18interface ArticleMedia {
19  kind: "image" | "video";
20  url: string;
21}
22 
23function resolveArticleMedia(mediaInfo: JsonObject): ArticleMedia | null {
24  const videoUrl = resolveBestXVideoVariantUrl(mediaInfo);
25  if (videoUrl) {
26    return {
27      kind: "video",
28      url: videoUrl,
29    };
30  }
31 
32  const rawUrl =
33    (typeof mediaInfo.original_img_url === "string" && mediaInfo.original_img_url) ||
34    (typeof mediaInfo.url === "string" && mediaInfo.url) ||
35    "";
36 
37  if (!rawUrl) {
38    return null;
39  }
40 
41  return {
42    kind: "image",
43    url: toHighResXImageUrl(rawUrl),
44  };
45}
46 
47function resolveArticleMediaUrl(mediaInfo: JsonObject): string {
48  return resolveArticleMedia(mediaInfo)?.url ?? "";
49}
50 
51function normalizeEntityMap(entityMap: unknown): Map<string, JsonObject> {
52  const normalized = new Map<string, JsonObject>();
53 
54  if (Array.isArray(entityMap)) {
55    for (const entry of entityMap) {
56      if (!isRecord(entry)) {
57        continue;
58      }
59 
60      const key =
61        typeof entry.key === "string" || typeof entry.key === "number"
62          ? String(entry.key)
63          : undefined;
64      const value = isRecord(entry.value) ? entry.value : undefined;
65      if (!key || !value) {
66        continue;
67      }
68      normalized.set(key, value);
69    }
70 
71    return normalized;
72  }
73 
74  if (!isRecord(entityMap)) {
75    return normalized;
76  }
77 
78  for (const [key, value] of Object.entries(entityMap)) {
79    if (!isRecord(value)) {
80      continue;
81    }
82    normalized.set(key, value);
83  }
84 
85  return normalized;
86}
87 
88function getEntityMarkdown(entityMap: Map<string, JsonObject>, entityKey: unknown): string | null {
89  const key =
90    typeof entityKey === "string" || typeof entityKey === "number"
91      ? String(entityKey)
92      : undefined;
93  if (!key) {
94    return null;
95  }
96 
97  const entity = entityMap.get(key);
98  if (!entity || entity.type !== "MARKDOWN") {
99    return null;
100  }
101 
102  const data = isRecord(entity.data) ? entity.data : {};
103  if (typeof data.markdown !== "string") {
104    return null;
105  }
106 
107  const markdown = data.markdown.trim();
108  return markdown || null;
109}
110 
111function getLinkUrl(entityMap: Map<string, JsonObject>, entityKey: unknown): string | null {
112  const key =
113    typeof entityKey === "string" || typeof entityKey === "number"
114      ? String(entityKey)
115      : undefined;
116  if (!key) {
117    return null;
118  }
119 
120  const entity = entityMap.get(key);
121  if (!entity || entity.type !== "LINK") {
122    return null;
123  }
124 
125  const data = isRecord(entity.data) ? entity.data : {};
126  const candidates = [
127    data.expanded_url,
128    data.expandedUrl,
129    data.original_url,
130    data.originalUrl,
131    data.url,
132    data.display_url,
133    data.displayUrl,
134  ];
135 
136  for (const candidate of candidates) {
137    if (typeof candidate === "string" && candidate.trim()) {
138      return candidate.trim();
139    }
140  }
141 
142  return null;
143}
144 
145function getTweetId(entityMap: Map<string, JsonObject>, entityKey: unknown): string | null {
146  const key =
147    typeof entityKey === "string" || typeof entityKey === "number"
148      ? String(entityKey)
149      : undefined;
150  if (!key) {
151    return null;
152  }
153 
154  const entity = entityMap.get(key);
155  if (!entity || entity.type !== "TWEET") {
156    return null;
157  }
158 
159  const data = isRecord(entity.data) ? entity.data : {};
160  if (typeof data.tweetId !== "string") {
161    return null;
162  }
163 
164  return data.tweetId;
165}
166 
167function buildMediaMap(articleResult: JsonObject): Map<string, ArticleMedia> {
168  const mediaMap = new Map<string, ArticleMedia>();
169  const mediaEntities = Array.isArray(articleResult.media_entities) ? articleResult.media_entities : [];
170 
171  for (const entity of mediaEntities) {
172    if (!isRecord(entity) || typeof entity.media_id !== "string" || !isRecord(entity.media_info)) {
173      continue;
174    }
175 
176    const media = resolveArticleMedia(entity.media_info);
177    if (media) {
178      mediaMap.set(entity.media_id, media);
179    }
180  }
181 
182  const coverMedia = isRecord(articleResult.cover_media) ? articleResult.cover_media : null;
183  if (coverMedia && typeof coverMedia.media_id === "string" && isRecord(coverMedia.media_info)) {
184    const media = resolveArticleMedia(coverMedia.media_info);
185    if (media) {
186      mediaMap.set(coverMedia.media_id, media);
187    }
188  }
189 
190  return mediaMap;
191}
192 
193function getMediaMarkdown(
194  entityMap: Map<string, JsonObject>,
195  entityKey: unknown,
196  mediaMap: Map<string, ArticleMedia>,
197): string[] {
198  const key =
199    typeof entityKey === "string" || typeof entityKey === "number"
200      ? String(entityKey)
201      : undefined;
202  if (!key) {
203    return [];
204  }
205 
206  const entity = entityMap.get(key);
207  if (!entity || entity.type !== "MEDIA") {
208    return [];
209  }
210 
211  const data = isRecord(entity.data) ? entity.data : {};
212  const mediaItems = Array.isArray(data.mediaItems) ? data.mediaItems : [];
213  const media: ArticleMedia[] = [];
214 
215  for (const item of mediaItems) {
216    if (!isRecord(item) || typeof item.mediaId !== "string") {
217      continue;
218    }
219    const mediaItem = mediaMap.get(item.mediaId);
220    if (mediaItem && !media.some((value) => value.url === mediaItem.url)) {
221      media.push(mediaItem);
222    }
223  }
224 
225  return media.map((item) => item.kind === "image" ? `![](${item.url})` : `[video](${item.url})`);
226}
227 
228function resolveTweetMarkdown(payloads: unknown[], tweetId: string, pageUrl: string): string | null {
229  for (const payload of payloads) {
230    const tweet = findTweetNodeById(payload, tweetId);
231    if (!tweet) {
232      continue;
233    }
234 
235    const xTweet = toXTweet(tweet, pageUrl);
236    const author = formatTweetAuthor(xTweet) ?? xTweet.url;
237    const lines = [`> ${author}`, ...xTweet.text.split("\n").map((line) => `> ${line}`)];
238 
239    const media = formatMediaList(xTweet.media).map((line) =>
240      line.startsWith("photo: ") ? `> ![](${line.slice("photo: ".length)})` : `> - ${line}`,
241    );
242 
243    const parts = [lines.join("\n")];
244    if (media.length > 0) {
245      parts.push([">", ...media].join("\n"));
246    }
247    parts.push(`> ${xTweet.url}`);
248 
249    return parts.join("\n").trim();
250  }
251 
252  return `> Embedded tweet: https://x.com/i/status/${tweetId}`;
253}
254 
255function replaceLinkEntities(text: string, block: JsonObject, entityMap: Map<string, JsonObject>): string {
256  const entityRanges = Array.isArray(block.entityRanges) ? block.entityRanges : [];
257  const replacements = entityRanges
258    .filter((range): range is JsonObject => isRecord(range))
259    .map((range) => {
260      const offset = typeof range.offset === "number" ? range.offset : -1;
261      const length = typeof range.length === "number" ? range.length : -1;
262      const url = getLinkUrl(entityMap, range.key);
263      return { offset, length, url };
264    })
265    .filter((range) => range.offset >= 0 && range.length > 0 && range.url)
266    .sort((left, right) => right.offset - left.offset);
267 
268  let next = text;
269  for (const replacement of replacements) {
270    next =
271      next.slice(0, replacement.offset) +
272      replacement.url +
273      next.slice(replacement.offset + replacement.length);
274  }
275  return next;
276}
277 
278function renderAtomicBlock(
279  block: JsonObject,
280  entityMap: Map<string, JsonObject>,
281  mediaMap: Map<string, ArticleMedia>,
282  payloads: unknown[],
283  pageUrl: string,
284): string | null {
285  const entityRanges = Array.isArray(block.entityRanges) ? block.entityRanges : [];
286  const parts: string[] = [];
287 
288  for (const range of entityRanges) {
289    if (!isRecord(range)) {
290      continue;
291    }
292 
293    const markdown = getEntityMarkdown(entityMap, range.key);
294    if (markdown) {
295      parts.push(markdown);
296      continue;
297    }
298 
299    const mediaMarkdown = getMediaMarkdown(entityMap, range.key, mediaMap);
300    if (mediaMarkdown.length > 0) {
301      parts.push(mediaMarkdown.join("\n\n"));
302      continue;
303    }
304 
305    const tweetId = getTweetId(entityMap, range.key);
306    if (tweetId) {
307      const tweetMarkdown = resolveTweetMarkdown(payloads, tweetId, pageUrl);
308      if (tweetMarkdown) {
309        parts.push(tweetMarkdown);
310      }
311    }
312  }
313 
314  if (parts.length === 0) {
315    return null;
316  }
317 
318  return parts.join("\n\n");
319}
320 
321function renderArticleBlocks(
322  blocks: unknown[],
323  entityMap: Map<string, JsonObject>,
324  mediaMap: Map<string, ArticleMedia>,
325  payloads: unknown[],
326  pageUrl: string,
327): string {
328  const parts: string[] = [];
329  let orderedCounter = 0;
330 
331  for (const block of blocks) {
332    if (!isRecord(block)) {
333      continue;
334    }
335 
336    const blockType = typeof block.type === "string" ? block.type : "unstyled";
337    const rawText = typeof block.text === "string" ? block.text : "";
338    const text = replaceLinkEntities(rawText, block, entityMap).trim();
339    if (!text && blockType !== "atomic") {
340      continue;
341    }
342 
343    if (blockType !== "ordered-list-item") {
344      orderedCounter = 0;
345    }
346 
347    switch (blockType) {
348      case "header-one":
349        parts.push(`# ${text}`);
350        break;
351      case "header-two":
352        parts.push(`## ${text}`);
353        break;
354      case "header-three":
355        parts.push(`### ${text}`);
356        break;
357      case "blockquote":
358        parts.push(`> ${text}`);
359        break;
360      case "unordered-list-item":
361        parts.push(`- ${text}`);
362        break;
363      case "ordered-list-item":
364        orderedCounter += 1;
365        parts.push(`${orderedCounter}. ${text}`);
366        break;
367      case "code-block":
368        parts.push(`\`\`\`\n${text}\n\`\`\``);
369        break;
370      case "atomic": {
371        const markdown = renderAtomicBlock(block, entityMap, mediaMap, payloads, pageUrl);
372        if (markdown) {
373          parts.push(markdown);
374        }
375        break;
376      }
377      default:
378        parts.push(text);
379        break;
380    }
381  }
382 
383  return parts.join("\n\n").trim();
384}
385 
386function getArticleResult(tweet: JsonObject): JsonObject | null {
387  if (
388    isRecord(tweet.article) &&
389    isRecord(tweet.article.article_results) &&
390    isRecord(tweet.article.article_results.result)
391  ) {
392    return tweet.article.article_results.result as JsonObject;
393  }
394  return null;
395}
396 
397function extractSummary(markdown: string): string | undefined {
398  const segments = markdown
399    .split(/\n\n+/)
400    .map((segment) => segment.trim())
401    .filter(Boolean);
402 
403  const preferred = segments.find((segment) => !/^(#|>|- |\d+\. |\`\`\`)/.test(segment));
404  return preferred?.slice(0, 220);
405}
406 
407export function extractArticleDocumentFromPayload(
408  payload: unknown,
409  statusId: string,
410  pageUrl: string,
411  payloads: unknown[] = [payload],
412): ExtractedDocument | null {
413  const tweet = findTweetNode(payload, statusId);
414  if (!tweet) {
415    return null;
416  }
417 
418  const articleResult = getArticleResult(tweet);
419  if (!articleResult) {
420    return null;
421  }
422 
423  const title = typeof articleResult.title === "string" ? articleResult.title.trim() : undefined;
424  const contentState = isRecord(articleResult.content_state) ? articleResult.content_state : {};
425  const blocks = Array.isArray(contentState.blocks) ? contentState.blocks : [];
426  const entityMap = normalizeEntityMap(contentState.entityMap);
427  const mediaMap = buildMediaMap(articleResult);
428  const richMarkdown = renderArticleBlocks(blocks, entityMap, mediaMap, payloads, pageUrl);
429  const plainText = typeof articleResult.plain_text === "string" ? articleResult.plain_text.trim() : "";
430  const markdown = richMarkdown || plainText || getTweetText(tweet);
431  if (!markdown) {
432    return null;
433  }
434 
435  const xTweet = toXTweet(tweet, pageUrl);
436  const user = getUser(tweet);
437  const coverMedia = isRecord(articleResult.cover_media) ? articleResult.cover_media : null;
438  const coverMediaInfo = coverMedia && isRecord(coverMedia.media_info) ? coverMedia.media_info : null;
439  const coverImage = coverMediaInfo ? resolveArticleMediaUrl(coverMediaInfo) || undefined : undefined;
440 
441  return {
442    url: pageUrl,
443    canonicalUrl: xTweet.url,
444    title: title || normalizeTitle(xTweet.text, "X Article"),
445    author: formatTweetAuthor(xTweet),
446    siteName: "X",
447    publishedAt: xTweet.createdAt,
448    summary: extractSummary(markdown) || xTweet.text.slice(0, 200) || undefined,
449    adapter: "x",
450    metadata: {
451      kind: "x/article",
452      tweetId: xTweet.id,
453      coverImage,
454      authorName: xTweet.authorName ?? user.name,
455      authorUsername: xTweet.author ?? user.screenName,
456      authorUrl: (xTweet.author ?? user.screenName) ? `https://x.com/${xTweet.author ?? user.screenName}` : undefined,
457      ...getTweetAuthorMetadata(xTweet),
458    },
459    content: [{ type: "markdown", markdown }],
460  };
461}
462