1import type { ExtractedDocument } from "../../extract/document";
2import {
3  formatMediaList,
4  formatTweetAuthor,
5  getLegacy,
6  getTweetAuthorMetadata,
7  isRecord,
8  normalizeTitle,
9  toXTweet,
10  unwrapTweetResult,
11} from "./shared";
12import type { JsonObject, XQuotedTweet, XTweet } from "./types";
13 
14interface ParsedThreadTweet extends XTweet {
15  userId?: string;
16  conversationId?: string;
17  inReplyToUserId?: string;
18  sortTimestamp: number;
19}
20 
21function compareTweetIds(left: string, right: string): number {
22  try {
23    const leftId = BigInt(left);
24    const rightId = BigInt(right);
25    if (leftId === rightId) {
26      return 0;
27    }
28    return leftId < rightId ? -1 : 1;
29  } catch {
30    return left.localeCompare(right);
31  }
32}
33 
34function toTimestamp(value: string | undefined): number {
35  if (!value) {
36    return 0;
37  }
38  const parsed = Date.parse(value);
39  return Number.isNaN(parsed) ? 0 : parsed;
40}
41 
42function scoreParsedTweet(tweet: ParsedThreadTweet): number {
43  return (
44    (tweet.text ? 4 : 0) +
45    (tweet.author ? 2 : 0) +
46    (tweet.authorName ? 2 : 0) +
47    (tweet.media.length > 0 ? 1 : 0)
48  );
49}
50 
51function toParsedThreadTweet(tweet: JsonObject, pageUrl: string): ParsedThreadTweet {
52  const legacy = getLegacy(tweet);
53  const xTweet = toXTweet(tweet, pageUrl);
54 
55  return {
56    ...xTweet,
57    userId: typeof legacy.user_id_str === "string" ? legacy.user_id_str : undefined,
58    conversationId: typeof legacy.conversation_id_str === "string" ? legacy.conversation_id_str : undefined,
59    inReplyToUserId: typeof legacy.in_reply_to_user_id_str === "string" ? legacy.in_reply_to_user_id_str : undefined,
60    sortTimestamp: toTimestamp(xTweet.createdAt),
61  };
62}
63 
64function collectTweetFromItemContent(
65  itemContent: unknown,
66  pageUrl: string,
67  tweets: Map<string, ParsedThreadTweet>,
68): void {
69  if (!isRecord(itemContent)) {
70    return;
71  }
72 
73  const tweet = unwrapTweetResult(
74    isRecord(itemContent.tweet_results) ? itemContent.tweet_results.result : null,
75  );
76  if (!tweet || typeof tweet.rest_id !== "string") {
77    return;
78  }
79 
80  const parsed = toParsedThreadTweet(tweet, pageUrl);
81  const existing = tweets.get(parsed.id);
82  if (!existing || scoreParsedTweet(parsed) >= scoreParsedTweet(existing)) {
83    tweets.set(parsed.id, parsed);
84  }
85}
86 
87function collectTweetsFromItems(
88  items: unknown,
89  pageUrl: string,
90  tweets: Map<string, ParsedThreadTweet>,
91): void {
92  if (!Array.isArray(items)) {
93    return;
94  }
95 
96  for (const item of items) {
97    if (!isRecord(item)) {
98      continue;
99    }
100 
101    if (isRecord(item.item) && isRecord(item.item.itemContent)) {
102      collectTweetFromItemContent(item.item.itemContent, pageUrl, tweets);
103      continue;
104    }
105 
106    if (isRecord(item.itemContent)) {
107      collectTweetFromItemContent(item.itemContent, pageUrl, tweets);
108    }
109  }
110}
111 
112function getInstructions(payload: unknown): unknown[] {
113  if (!isRecord(payload) || !isRecord(payload.data)) {
114    return [];
115  }
116 
117  const { data } = payload;
118  return (
119    (isRecord(data.threaded_conversation_with_injections_v2) &&
120    Array.isArray(data.threaded_conversation_with_injections_v2.instructions)
121      ? data.threaded_conversation_with_injections_v2.instructions
122      : undefined) ??
123    (isRecord(data.threaded_conversation_with_injections) &&
124    Array.isArray(data.threaded_conversation_with_injections.instructions)
125      ? data.threaded_conversation_with_injections.instructions
126      : undefined) ??
127    (isRecord(data.tweetResult) &&
128    isRecord(data.tweetResult.result) &&
129    isRecord(data.tweetResult.result.timeline) &&
130    Array.isArray(data.tweetResult.result.timeline.instructions)
131      ? data.tweetResult.result.timeline.instructions
132      : [])
133  );
134}
135 
136function parseTweetDetailPayload(payload: unknown, pageUrl: string): ParsedThreadTweet[] {
137  const tweets = new Map<string, ParsedThreadTweet>();
138 
139  const instructions = getInstructions(payload);
140  for (const instruction of instructions) {
141    if (!isRecord(instruction)) {
142      continue;
143    }
144 
145    collectTweetsFromItems(instruction.moduleItems, pageUrl, tweets);
146 
147    if (!Array.isArray(instruction.entries)) {
148      continue;
149    }
150 
151    for (const entry of instruction.entries) {
152      if (!isRecord(entry)) {
153        continue;
154      }
155 
156      const content = isRecord(entry.content) ? entry.content : {};
157      collectTweetFromItemContent(content.itemContent, pageUrl, tweets);
158      collectTweetsFromItems(content.items, pageUrl, tweets);
159    }
160  }
161 
162  return Array.from(tweets.values());
163}
164 
165function buildContinuousThread(tweets: ParsedThreadTweet[], statusId: string): ParsedThreadTweet[] {
166  const byId = new Map<string, ParsedThreadTweet>();
167  for (const tweet of tweets) {
168    const existing = byId.get(tweet.id);
169    if (!existing || scoreParsedTweet(tweet) >= scoreParsedTweet(existing)) {
170      byId.set(tweet.id, tweet);
171    }
172  }
173 
174  const rootTweet = byId.get(statusId);
175  if (!rootTweet?.userId || !rootTweet.conversationId) {
176    return [];
177  }
178 
179  const candidates = Array.from(byId.values()).filter(
180    (tweet) =>
181      tweet.id === statusId ||
182      (tweet.userId === rootTweet.userId && tweet.conversationId === rootTweet.conversationId),
183  );
184 
185  const repliesByParent = new Map<string, ParsedThreadTweet[]>();
186  for (const tweet of candidates) {
187    if (!tweet.inReplyTo || tweet.id === statusId) {
188      continue;
189    }
190    const bucket = repliesByParent.get(tweet.inReplyTo) ?? [];
191    bucket.push(tweet);
192    bucket.sort((left, right) => {
193      if (left.sortTimestamp !== right.sortTimestamp) {
194        return left.sortTimestamp - right.sortTimestamp;
195      }
196      return compareTweetIds(left.id, right.id);
197    });
198    repliesByParent.set(tweet.inReplyTo, bucket);
199  }
200 
201  const ancestorPath: ParsedThreadTweet[] = [rootTweet];
202  const ancestorSeen = new Set<string>([rootTweet.id]);
203  let currentAncestor = rootTweet;
204 
205  while (currentAncestor.inReplyTo) {
206    const parent = byId.get(currentAncestor.inReplyTo);
207    if (!parent || ancestorSeen.has(parent.id)) {
208      break;
209    }
210    ancestorPath.unshift(parent);
211    ancestorSeen.add(parent.id);
212    currentAncestor = parent;
213  }
214 
215  const chain = ancestorPath.slice();
216  const seen = new Set<string>(chain.map((tweet) => tweet.id));
217  let currentId = rootTweet.id;
218 
219  while (true) {
220    const next = (repliesByParent.get(currentId) ?? []).find((tweet) => !seen.has(tweet.id));
221    if (!next) {
222      break;
223    }
224    chain.push(next);
225    seen.add(next.id);
226    currentId = next.id;
227  }
228 
229  return chain;
230}
231 
232export function extractThreadTweetsFromPayloads(
233  payloads: unknown[],
234  statusId: string,
235  pageUrl: string,
236): XTweet[] {
237  const parsedTweets: ParsedThreadTweet[] = [];
238 
239  for (const payload of payloads) {
240    parsedTweets.push(...parseTweetDetailPayload(payload, pageUrl));
241  }
242 
243  return buildContinuousThread(parsedTweets, statusId).map(({ sortTimestamp: _sortTimestamp, ...tweet }) => tweet);
244}
245 
246function buildQuotedTweetMarkdown(quotedTweet: XQuotedTweet): string {
247  const author = quotedTweet.author ? `@${quotedTweet.author}` : "Unknown";
248  const name = quotedTweet.authorName ? `${quotedTweet.authorName} ` : "";
249  const lines: string[] = [`Quoted Tweet${quotedTweet.author || quotedTweet.authorName ? `: ${name}${author}`.trim() : ""}`];
250 
251  if (quotedTweet.text) {
252    lines.push(...quotedTweet.text.split("\n"));
253  }
254 
255  for (const mediaLine of formatMediaList(quotedTweet.media)) {
256    lines.push(mediaLine);
257  }
258 
259  return lines.map((line) => (line ? `> ${line}` : ">")).join("\n");
260}
261 
262function buildThreadMarkdown(tweets: XTweet[]): string {
263  return tweets
264    .map((tweet, index) => {
265      const lines: string[] = [];
266      const author = tweet.author ? `@${tweet.author}` : "Unknown";
267      const name = tweet.authorName ? `${tweet.authorName} ` : "";
268      lines.push(`## ${index + 1}. ${name}${author}`.trim());
269      if (tweet.createdAt) {
270        lines.push(`_Published: ${tweet.createdAt}_`);
271      }
272      lines.push(tweet.text || "(No text)");
273      const mediaLines = formatMediaList(tweet.media);
274      if (mediaLines.length > 0) {
275        lines.push(mediaLines.map((line) => `- ${line}`).join("\n"));
276      }
277      if (tweet.quotedTweet) {
278        lines.push(buildQuotedTweetMarkdown(tweet.quotedTweet));
279      }
280      return lines.join("\n\n");
281    })
282    .join("\n\n");
283}
284 
285export function extractThreadDocumentFromPayloads(
286  payloads: unknown[],
287  statusId: string,
288  pageUrl: string,
289): ExtractedDocument | null {
290  const tweets = extractThreadTweetsFromPayloads(payloads, statusId, pageUrl);
291  if (tweets.length <= 1) {
292    return null;
293  }
294 
295  const rootTweet = tweets[0];
296  const rootAuthor = formatTweetAuthor(rootTweet);
297 
298  return {
299    url: pageUrl,
300    canonicalUrl: rootTweet.url,
301    title: normalizeTitle(rootTweet.text, "X Thread"),
302    author: rootAuthor,
303    siteName: "X",
304    publishedAt: rootTweet.createdAt,
305    summary: rootTweet.text.slice(0, 200) || undefined,
306    adapter: "x",
307    metadata: {
308      kind: "x/thread",
309      tweetId: rootTweet.id,
310      tweetCount: tweets.length,
311      lastTweetId: tweets[tweets.length - 1]?.id,
312      ...getTweetAuthorMetadata(rootTweet),
313    },
314    content: [{ type: "markdown", markdown: buildThreadMarkdown(tweets) }],
315  };
316}
317