1import fs from "node:fs";
2import path from "node:path";
3import readline from "node:readline";
4import process from "node:process";
5import { mkdir, readFile, writeFile } from "node:fs/promises";
6 
7import { fetchXArticle } from "./graphql.js";
8import { formatArticleMarkdown } from "./markdown.js";
9import { localizeMarkdownMedia, type LocalizeMarkdownMediaResult } from "./media-localizer.js";
10import { resolveReferencedTweetsFromArticle } from "./referenced-tweets.js";
11import { hasRequiredXCookies, loadXCookies, refreshXCookies } from "./cookies.js";
12import { resolveXToMarkdownConsentPath } from "./paths.js";
13import { tweetToMarkdown } from "./tweet-to-markdown.js";
14 
15type CliArgs = {
16  url: string | null;
17  output: string | null;
18  json: boolean;
19  login: boolean;
20  downloadMedia: boolean;
21  help: boolean;
22};
23 
24type ConsentRecord = {
25  version: number;
26  accepted: boolean;
27  acceptedAt: string;
28  disclaimerVersion: string;
29};
30 
31const DISCLAIMER_VERSION = "1.0";
32 
33function formatScriptCommand(fallback: string): string {
34  const raw = process.argv[1];
35  const displayPath = raw
36    ? (() => {
37        const relative = path.relative(process.cwd(), raw);
38        return relative && !relative.startsWith("..") ? relative : raw;
39      })()
40    : fallback;
41  const quotedPath = displayPath.includes(" ")
42    ? `"${displayPath.replace(/"/g, '\\"')}"`
43    : displayPath;
44  return `npx -y bun ${quotedPath}`;
45}
46 
47function printUsage(exitCode: number): never {
48  const cmd = formatScriptCommand("scripts/main.ts");
49  console.log(`X (Twitter) to Markdown
50 
51Usage:
52  ${cmd} <url>
53  ${cmd} --url <url>
54 
55Options:
56  --output <path>, -o  Output path (file or dir). Default: ./x-to-markdown/<slug>/
57  --json               Output as JSON
58  --download-media     Download images/videos to local ./imgs and ./videos next to markdown
59  --login              Refresh cookies only, then exit
60  --help, -h           Show help
61 
62Examples:
63  ${cmd} https://x.com/username/status/1234567890
64  ${cmd} https://x.com/i/article/1234567890 -o ./article.md
65  ${cmd} https://x.com/username/status/1234567890 -o ./out/
66  ${cmd} https://x.com/username/status/1234567890 --download-media
67  ${cmd} https://x.com/username/status/1234567890 --json | jq -r '.markdownPath'
68  ${cmd} --login
69`);
70  process.exit(exitCode);
71}
72 
73function parseArgs(argv: string[]): CliArgs {
74  const out: CliArgs = {
75    url: null,
76    output: null,
77    json: false,
78    login: false,
79    downloadMedia: false,
80    help: false,
81  };
82 
83  const positional: string[] = [];
84 
85  for (let i = 0; i < argv.length; i++) {
86    const a = argv[i]!;
87 
88    if (a === "--help" || a === "-h") {
89      out.help = true;
90      continue;
91    }
92 
93    if (a === "--json") {
94      out.json = true;
95      continue;
96    }
97 
98    if (a === "--login") {
99      out.login = true;
100      continue;
101    }
102 
103    if (a === "--download-media") {
104      out.downloadMedia = true;
105      continue;
106    }
107 
108    if (a === "--url") {
109      const v = argv[++i];
110      if (!v) throw new Error("Missing value for --url");
111      out.url = v;
112      continue;
113    }
114 
115    if (a === "--output" || a === "-o") {
116      const v = argv[++i];
117      if (!v) throw new Error(`Missing value for ${a}`);
118      out.output = v;
119      continue;
120    }
121 
122    if (a.startsWith("-")) {
123      throw new Error(`Unknown option: ${a}`);
124    }
125 
126    positional.push(a);
127  }
128 
129  if (!out.url && positional.length > 0) {
130    out.url = positional[0]!;
131  }
132 
133  return out;
134}
135 
136function normalizeInputUrl(input: string): string {
137  const trimmed = input.trim();
138  if (!trimmed) return "";
139  try {
140    return new URL(trimmed).toString();
141  } catch {
142    return trimmed;
143  }
144}
145 
146function parseArticleId(input: string): string | null {
147  const trimmed = input.trim();
148  if (!trimmed) return null;
149 
150  try {
151    const parsed = new URL(trimmed);
152    const match = parsed.pathname.match(/\/(?:i\/)?article\/(\d+)/);
153    if (match?.[1]) return match[1];
154  } catch {
155    return null;
156  }
157 
158  return null;
159}
160 
161function parseTweetId(input: string): string | null {
162  const trimmed = input.trim();
163  if (!trimmed) return null;
164  if (/^\d+$/.test(trimmed)) return trimmed;
165 
166  try {
167    const parsed = new URL(trimmed);
168    const match = parsed.pathname.match(/\/status(?:es)?\/(\d+)/);
169    if (match?.[1]) return match[1];
170  } catch {
171    return null;
172  }
173 
174  return null;
175}
176 
177function parseTweetUsername(input: string): string | null {
178  const trimmed = input.trim();
179  if (!trimmed) return null;
180  try {
181    const parsed = new URL(trimmed);
182    const match = parsed.pathname.match(/^\/([^/]+)\/status(?:es)?\/\d+/);
183    if (match?.[1]) return match[1];
184  } catch {
185    return null;
186  }
187  return null;
188}
189 
190function sanitizeSlug(input: string): string {
191  return input
192    .trim()
193    .replace(/^@/, "")
194    .replace(/[^a-zA-Z0-9_-]+/g, "-")
195    .replace(/-+/g, "-")
196    .replace(/^[-_]+|[-_]+$/g, "")
197    .slice(0, 120);
198}
199 
200function extractContentSlug(markdown: string): string {
201  const headingMatch = markdown.match(/^#\s+(.+)$/m);
202  if (headingMatch?.[1]) {
203    return sanitizeSlug(headingMatch[1].slice(0, 60)).toLowerCase();
204  }
205  const lines = markdown.split("\n");
206  let inFrontmatter = false;
207  for (const line of lines) {
208    if (line === "---") {
209      inFrontmatter = !inFrontmatter;
210      continue;
211    }
212    if (inFrontmatter) continue;
213    const trimmed = line.trim();
214    if (trimmed) {
215      return sanitizeSlug(trimmed.slice(0, 60)).toLowerCase();
216    }
217  }
218  return "untitled";
219}
220 
221function resolveSlugAndId(normalizedUrl: string, kind: "tweet" | "article"): { slug: string; idPart: string } {
222  const articleId = kind === "article" ? parseArticleId(normalizedUrl) : null;
223  const tweetId = kind === "tweet" ? parseTweetId(normalizedUrl) : null;
224  const username = kind === "tweet" ? parseTweetUsername(normalizedUrl) : null;
225 
226  const idPart = articleId ?? tweetId ?? String(Date.now());
227  const userSlug = username ? sanitizeSlug(username) : null;
228  const slug = userSlug ?? idPart;
229  return { slug, idPart };
230}
231 
232function extractFrontmatterUrls(markdown: string): string[] {
233  const match = markdown.match(/^---\n([\s\S]*?)\n---/);
234  if (!match?.[1]) return [];
235 
236  const lines = match[1].split("\n");
237  const urls: string[] = [];
238  for (const line of lines) {
239    const m = line.match(/^(url|requestedUrl):\s*["']([^"']+)["']\s*$/);
240    if (m?.[2]) {
241      urls.push(m[2]);
242    }
243  }
244  return urls;
245}
246 
247function frontmatterMatchesTarget(
248  markdown: string,
249  normalizedUrl: string,
250  kind: "tweet" | "article"
251): boolean {
252  const urls = extractFrontmatterUrls(markdown);
253  if (urls.length === 0) return false;
254 
255  const targetId = kind === "article" ? parseArticleId(normalizedUrl) : parseTweetId(normalizedUrl);
256  if (!targetId) return false;
257 
258  for (const url of urls) {
259    const candidateId = kind === "article" ? parseArticleId(url) : parseTweetId(url);
260    if (candidateId && candidateId === targetId) {
261      return true;
262    }
263  }
264 
265  return false;
266}
267 
268function listMarkdownFiles(dirPath: string): string[] {
269  try {
270    return fs
271      .readdirSync(dirPath)
272      .filter((name) => name.toLowerCase().endsWith(".md"))
273      .map((name) => path.join(dirPath, name))
274      .sort();
275  } catch {
276    return [];
277  }
278}
279 
280function resolveExistingMarkdownPath(
281  normalizedUrl: string,
282  kind: "tweet" | "article",
283  argsOutput: string | null
284): string | null {
285  const { slug, idPart } = resolveSlugAndId(normalizedUrl, kind);
286  const candidateDirs = new Set<string>();
287  const candidateFiles = new Set<string>();
288 
289  if (argsOutput) {
290    const resolved = path.resolve(argsOutput);
291    const looksDir = argsOutput.endsWith("/") || argsOutput.endsWith("\\");
292    try {
293      if (fs.existsSync(resolved)) {
294        const stat = fs.statSync(resolved);
295        if (stat.isFile()) {
296          candidateFiles.add(resolved);
297        } else if (stat.isDirectory()) {
298          candidateDirs.add(path.join(resolved, slug, idPart));
299          candidateDirs.add(resolved);
300        }
301      } else if (looksDir) {
302        candidateDirs.add(path.join(resolved, slug, idPart));
303      }
304    } catch {
305      // ignore and continue
306    }
307  } else {
308    candidateDirs.add(path.resolve(process.cwd(), "x-to-markdown", slug, idPart));
309  }
310 
311  for (const filePath of candidateFiles) {
312    if (!filePath.toLowerCase().endsWith(".md")) continue;
313    try {
314      const markdown = fs.readFileSync(filePath, "utf8");
315      if (frontmatterMatchesTarget(markdown, normalizedUrl, kind)) {
316        return filePath;
317      }
318    } catch {
319      // ignore and continue
320    }
321  }
322 
323  for (const dirPath of candidateDirs) {
324    if (!fs.existsSync(dirPath)) continue;
325    let stat: fs.Stats;
326    try {
327      stat = fs.statSync(dirPath);
328    } catch {
329      continue;
330    }
331    if (!stat.isDirectory()) continue;
332 
333    const markdownFiles = listMarkdownFiles(dirPath);
334    for (const filePath of markdownFiles) {
335      try {
336        const markdown = fs.readFileSync(filePath, "utf8");
337        if (frontmatterMatchesTarget(markdown, normalizedUrl, kind)) {
338          return filePath;
339        }
340      } catch {
341        // ignore and continue
342      }
343    }
344  }
345 
346  return null;
347}
348 
349async function resolveOutputPath(
350  normalizedUrl: string,
351  kind: "tweet" | "article",
352  argsOutput: string | null,
353  contentSlug: string,
354  log: (message: string) => void
355): Promise<{ outputDir: string; markdownPath: string; slug: string }> {
356  const articleId = kind === "article" ? parseArticleId(normalizedUrl) : null;
357  const tweetId = kind === "tweet" ? parseTweetId(normalizedUrl) : null;
358  const username = kind === "tweet" ? parseTweetUsername(normalizedUrl) : null;
359 
360  const userSlug = username ? sanitizeSlug(username) : null;
361  const idPart = articleId ?? tweetId ?? String(Date.now());
362  const slug = userSlug ?? idPart;
363 
364  const defaultFileName = `${contentSlug}.md`;
365 
366  if (argsOutput) {
367    const wantsDir = argsOutput.endsWith("/") || argsOutput.endsWith("\\");
368    const resolved = path.resolve(argsOutput);
369    try {
370      if (wantsDir || (fs.existsSync(resolved) && fs.statSync(resolved).isDirectory())) {
371        const outputDir = path.join(resolved, slug, idPart);
372        await mkdir(outputDir, { recursive: true });
373        return { outputDir, markdownPath: path.join(outputDir, defaultFileName), slug };
374      }
375    } catch {
376      // treat as file path
377    }
378 
379    const outputDir = path.dirname(resolved);
380    await mkdir(outputDir, { recursive: true });
381    return { outputDir, markdownPath: resolved, slug };
382  }
383 
384  const outputDir = path.resolve(process.cwd(), "x-to-markdown", slug, idPart);
385  await mkdir(outputDir, { recursive: true });
386  return { outputDir, markdownPath: path.join(outputDir, defaultFileName), slug };
387}
388 
389function formatMetaMarkdown(meta: Record<string, string | number | null | undefined>): string {
390  const lines = ["---"];
391  for (const [key, value] of Object.entries(meta)) {
392    if (value === undefined || value === null || value === "") continue;
393    if (typeof value === "number") {
394      lines.push(`${key}: ${value}`);
395    } else {
396      lines.push(`${key}: ${JSON.stringify(value)}`);
397    }
398  }
399  lines.push("---");
400  return lines.join("\n");
401}
402 
403async function promptYesNo(question: string): Promise<boolean> {
404  if (!process.stdin.isTTY) return false;
405 
406  const rl = readline.createInterface({
407    input: process.stdin,
408    output: process.stderr,
409  });
410 
411  try {
412    const answer = await new Promise<string>((resolve) => rl.question(question, resolve));
413    const normalized = answer.trim().toLowerCase();
414    return normalized === "y" || normalized === "yes";
415  } finally {
416    rl.close();
417  }
418}
419 
420function isValidConsent(value: unknown): value is ConsentRecord {
421  if (!value || typeof value !== "object") return false;
422  const record = value as Partial<ConsentRecord>;
423  return (
424    record.accepted === true &&
425    record.disclaimerVersion === DISCLAIMER_VERSION &&
426    typeof record.acceptedAt === "string" &&
427    record.acceptedAt.length > 0
428  );
429}
430 
431async function ensureConsent(log: (message: string) => void): Promise<void> {
432  const consentPath = resolveXToMarkdownConsentPath();
433 
434  try {
435    if (fs.existsSync(consentPath) && fs.statSync(consentPath).isFile()) {
436      const raw = await readFile(consentPath, "utf8");
437      const parsed = JSON.parse(raw) as unknown;
438      if (isValidConsent(parsed)) {
439        log(
440          `⚠️  Warning: Using reverse-engineered X API (not official). Accepted on: ${(parsed as ConsentRecord).acceptedAt}`
441        );
442        return;
443      }
444    }
445  } catch {
446    // fall through to prompt
447  }
448 
449  log(`⚠️  DISCLAIMER
450 
451This tool uses a reverse-engineered X (Twitter) API, NOT an official API.
452 
453Risks:
454- May break without notice if X changes their API
455- No official support or guarantees
456- Account restrictions possible if API usage detected
457- Use at your own risk
458`);
459 
460  if (!process.stdin.isTTY) {
461    throw new Error(
462      `Consent required. Run in a TTY or create ${consentPath} with accepted: true and disclaimerVersion: ${DISCLAIMER_VERSION}`
463    );
464  }
465 
466  const accepted = await promptYesNo("Do you accept these terms and wish to continue? (y/N): ");
467  if (!accepted) {
468    throw new Error("User declined the disclaimer. Exiting.");
469  }
470 
471  await mkdir(path.dirname(consentPath), { recursive: true });
472  const payload: ConsentRecord = {
473    version: 1,
474    accepted: true,
475    acceptedAt: new Date().toISOString(),
476    disclaimerVersion: DISCLAIMER_VERSION,
477  };
478  await writeFile(consentPath, JSON.stringify(payload, null, 2), "utf8");
479  log(`[x-to-markdown] Consent saved to: ${consentPath}`);
480}
481 
482async function convertArticleToMarkdown(
483  inputUrl: string,
484  articleId: string,
485  log: (message: string) => void
486): Promise<string> {
487  log("[x-to-markdown] Loading cookies...");
488  const cookieMap = await loadXCookies(log);
489  if (!hasRequiredXCookies(cookieMap)) {
490    throw new Error("Missing auth cookies. Provide X_AUTH_TOKEN and X_CT0 or log in via Chrome.");
491  }
492 
493  log(`[x-to-markdown] Fetching article ${articleId}...`);
494  const article = await fetchXArticle(articleId, cookieMap, false);
495  const referencedTweets = await resolveReferencedTweetsFromArticle(article, cookieMap, { log });
496  const { markdown: body, coverUrl } = formatArticleMarkdown(article, { referencedTweets });
497 
498  const title = typeof (article as any)?.title === "string" ? String((article as any).title).trim() : "";
499  const meta = formatMetaMarkdown({
500    url: `https://x.com/i/article/${articleId}`,
501    requestedUrl: inputUrl,
502    title: title || null,
503    coverImage: coverUrl,
504  });
505 
506  return [meta, body.trimEnd()].filter(Boolean).join("\n\n").trimEnd();
507}
508 
509async function main(): Promise<void> {
510  const args = parseArgs(process.argv.slice(2));
511  if (args.help) printUsage(0);
512  if (!args.login && !args.url) printUsage(1);
513 
514  const log = (message: string) => console.error(message);
515  await ensureConsent(log);
516 
517  if (args.login) {
518    log("[x-to-markdown] Refreshing cookies via browser login...");
519    const cookieMap = await refreshXCookies(log);
520    if (!hasRequiredXCookies(cookieMap)) {
521      throw new Error("Missing auth cookies after login. Please ensure you are logged in to X.");
522    }
523    log("[x-to-markdown] Cookies refreshed.");
524    return;
525  }
526 
527  const normalizedUrl = normalizeInputUrl(args.url ?? "");
528  const articleId = parseArticleId(normalizedUrl);
529  const tweetId = parseTweetId(normalizedUrl);
530  if (!articleId && !tweetId) {
531    throw new Error("Invalid X url. Examples: https://x.com/<user>/status/<id> or https://x.com/i/article/<id>");
532  }
533 
534  const kind = articleId ? ("article" as const) : ("tweet" as const);
535 
536  if (args.downloadMedia) {
537    const existingMarkdownPath = resolveExistingMarkdownPath(normalizedUrl, kind, args.output);
538    if (existingMarkdownPath) {
539      log(`[x-to-markdown] Reusing existing markdown: ${existingMarkdownPath}`);
540      const existingMarkdown = await readFile(existingMarkdownPath, "utf8");
541      const mediaResult = await localizeMarkdownMedia(existingMarkdown, {
542        markdownPath: existingMarkdownPath,
543        log,
544      });
545      const didLocalize =
546        mediaResult.downloadedImages > 0 ||
547        mediaResult.downloadedVideos > 0 ||
548        mediaResult.markdown !== existingMarkdown;
549 
550      if (didLocalize) {
551        await writeFile(existingMarkdownPath, mediaResult.markdown, "utf8");
552        log(
553          `[x-to-markdown] Media localized: images=${mediaResult.downloadedImages}, videos=${mediaResult.downloadedVideos}`
554        );
555        log(`[x-to-markdown] Saved: ${existingMarkdownPath}`);
556 
557        const { slug } = resolveSlugAndId(normalizedUrl, kind);
558        if (args.json) {
559          console.log(
560            JSON.stringify(
561              {
562                url: articleId ? `https://x.com/i/article/${articleId}` : normalizedUrl,
563                requestedUrl: normalizedUrl,
564                type: kind,
565                slug,
566                outputDir: path.dirname(existingMarkdownPath),
567                markdownPath: existingMarkdownPath,
568                downloadMedia: true,
569                downloadedImages: mediaResult.downloadedImages,
570                downloadedVideos: mediaResult.downloadedVideos,
571                imageDir: mediaResult.imageDir,
572                videoDir: mediaResult.videoDir,
573              },
574              null,
575              2
576            )
577          );
578        } else {
579          console.log(existingMarkdownPath);
580        }
581        return;
582      }
583 
584      log("[x-to-markdown] Existing markdown already localized; rebuilding content to refresh placement.");
585    }
586  }
587 
588  let markdown =
589    kind === "article" && articleId
590      ? await convertArticleToMarkdown(normalizedUrl, articleId, log)
591      : await tweetToMarkdown(normalizedUrl, { log });
592 
593  const contentSlug = extractContentSlug(markdown);
594  const { outputDir, markdownPath, slug } = await resolveOutputPath(normalizedUrl, kind, args.output, contentSlug, log);
595 
596  let mediaResult: LocalizeMarkdownMediaResult | null = null;
597 
598  if (args.downloadMedia) {
599    mediaResult = await localizeMarkdownMedia(markdown, {
600      markdownPath,
601      log,
602    });
603    markdown = mediaResult.markdown;
604    log(
605      `[x-to-markdown] Media localized: images=${mediaResult.downloadedImages}, videos=${mediaResult.downloadedVideos}`
606    );
607  }
608 
609  await writeFile(markdownPath, markdown, "utf8");
610  log(`[x-to-markdown] Saved: ${markdownPath}`);
611 
612  if (args.json) {
613    console.log(
614      JSON.stringify(
615        {
616          url: articleId ? `https://x.com/i/article/${articleId}` : normalizedUrl,
617          requestedUrl: normalizedUrl,
618          type: kind,
619          slug,
620          outputDir,
621          markdownPath,
622          downloadMedia: args.downloadMedia,
623          downloadedImages: mediaResult?.downloadedImages ?? 0,
624          downloadedVideos: mediaResult?.downloadedVideos ?? 0,
625          imageDir: mediaResult?.imageDir ?? null,
626          videoDir: mediaResult?.videoDir ?? null,
627        },
628        null,
629        2
630      )
631    );
632  } else {
633    console.log(markdownPath);
634  }
635}
636 
637await main().catch((error) => {
638  console.error(error instanceof Error ? error.message : String(error ?? ""));
639  process.exit(1);
640});
641