1import { mkdir, writeFile } from "node:fs/promises";
2import { join } from "node:path";
3import { createInterface } from "node:readline";
4import { connectChrome, type ChromeConnection } from "../browser/chrome-launcher";
5import { CdpClient } from "../browser/cdp-client";
6import { detectInteractionGate } from "../browser/interaction-gates";
7import { NetworkJournal } from "../browser/network-journal";
8import { BrowserSession } from "../browser/session";
9import { genericAdapter, resolveAdapter } from "../adapters";
10import { isXSessionReady } from "../adapters/x/session";
11import type { ExtractedDocument } from "../extract/document";
12import { renderMarkdown } from "../extract/markdown-renderer";
13import { downloadMediaAssets } from "../media/default-downloader";
14import { rewriteMarkdownMediaLinks } from "../media/markdown-media";
15import { createLogger } from "../utils/logger";
16import { normalizeUrl } from "../utils/url";
17import type {
18  Adapter,
19  AdapterContext,
20  AdapterLoginInfo,
21  LoginState,
22  MediaAsset,
23  WaitForInteractionRequest,
24} from "../adapters/types";
25 
26export type WaitMode = "none" | "interaction" | "force";
27export type OutputFormat = "markdown" | "json";
28 
29export interface ConvertCommandOptions {
30  url?: string;
31  output?: string;
32  format: OutputFormat;
33  adapter?: string;
34  debugDir?: string;
35  cdpUrl?: string;
36  browserPath?: string;
37  chromeProfileDir?: string;
38  headless: boolean;
39  downloadMedia: boolean;
40  mediaDir?: string;
41  waitMode: WaitMode;
42  interactionTimeoutMs: number;
43  interactionPollIntervalMs: number;
44  timeoutMs: number;
45}
46 
47interface RuntimeResources {
48  chrome: ChromeConnection;
49  cdp: CdpClient;
50  browser: BrowserSession;
51  network: NetworkJournal;
52  interactive: boolean;
53}
54 
55interface ForceWaitSnapshot {
56  url: string;
57  hasGate: boolean;
58  loginState: LoginState | "unavailable";
59  sessionReady: boolean;
60}
61 
62interface SuccessfulConvertOutput {
63  adapter: string;
64  status: "ok";
65  login?: AdapterLoginInfo;
66  media: MediaAsset[];
67  downloads: Awaited<ReturnType<typeof downloadMediaAssets>> | null;
68  document: ExtractedDocument;
69  markdown: string;
70}
71 
72interface InteractionRequiredOutput {
73  adapter: string;
74  status: "needs_interaction";
75  login?: AdapterLoginInfo;
76  interaction: WaitForInteractionRequest;
77}
78 
79function sleep(ms: number): Promise<void> {
80  return new Promise((resolve) => setTimeout(resolve, ms));
81}
82 
83function isForceWaitSessionReady(snapshot: ForceWaitSnapshot): boolean {
84  return snapshot.sessionReady;
85}
86 
87export function shouldKeepBrowserOpenAfterInteraction(options: {
88  launched: boolean;
89  interaction: Pick<WaitForInteractionRequest, "kind" | "provider">;
90}): boolean {
91  return options.launched && options.interaction.kind === "login" && options.interaction.provider === "x";
92}
93 
94export function shouldAutoContinueForceWait(
95  initial: ForceWaitSnapshot,
96  current: ForceWaitSnapshot,
97): boolean {
98  if (initial.hasGate && !current.hasGate) {
99    return true;
100  }
101 
102  if (initial.loginState === "logged_out" && current.loginState !== "logged_out" && isForceWaitSessionReady(current)) {
103    return true;
104  }
105 
106  if (initial.loginState !== "logged_in" && current.loginState === "logged_in" && isForceWaitSessionReady(current)) {
107    return true;
108  }
109 
110  if (
111    current.url !== initial.url &&
112    !current.hasGate &&
113    current.loginState !== "logged_out" &&
114    isForceWaitSessionReady(current)
115  ) {
116    return true;
117  }
118 
119  return false;
120}
121 
122async function writeOutput(path: string, content: string): Promise<void> {
123  const directory = path.includes("/") ? path.slice(0, path.lastIndexOf("/")) : "";
124  if (directory) {
125    await mkdir(directory, { recursive: true });
126  }
127  await writeFile(path, content, "utf8");
128}
129 
130async function writeDebugArtifacts(
131  debugDir: string,
132  document: ExtractedDocument,
133  markdown: string,
134  browser: BrowserSession,
135  network: NetworkJournal,
136): Promise<void> {
137  await mkdir(debugDir, { recursive: true });
138 
139  const html = await browser.getHTML().catch(() => "");
140  const networkDump = await network.toJSON({ includeBodies: true });
141 
142  await Promise.all([
143    writeFile(join(debugDir, "document.json"), JSON.stringify(document, null, 2), "utf8"),
144    writeFile(join(debugDir, "markdown.md"), markdown, "utf8"),
145    writeFile(join(debugDir, "page.html"), html, "utf8"),
146    writeFile(join(debugDir, "network.json"), JSON.stringify(networkDump, null, 2), "utf8"),
147  ]);
148}
149 
150async function openRuntime(
151  options: ConvertCommandOptions,
152  interactive: boolean,
153  debugEnabled: boolean,
154): Promise<RuntimeResources> {
155  const logger = createLogger(debugEnabled);
156  if (interactive) {
157    logger.info("Opening Chrome in interactive mode.");
158  }
159  const chrome = await connectChrome({
160    cdpUrl: options.cdpUrl,
161    browserPath: options.browserPath,
162    profileDir: options.chromeProfileDir,
163    headless: interactive ? false : options.headless,
164    logger,
165  });
166 
167  const cdp = await CdpClient.connect(chrome.browserWsUrl);
168  const browser = await BrowserSession.open(cdp, { interactive });
169  if (interactive) {
170    await browser.bringToFront().catch(() => {});
171  }
172  const network = new NetworkJournal(browser.targetSession, logger);
173  await network.start();
174 
175  return {
176    chrome,
177    cdp,
178    browser,
179    network,
180    interactive,
181  };
182}
183 
184async function closeRuntime(runtime: RuntimeResources | null | undefined): Promise<void> {
185  if (!runtime) {
186    return;
187  }
188  runtime.network.stop();
189  await runtime.browser.close().catch(() => {});
190  await runtime.cdp.close().catch(() => {});
191  await runtime.chrome.close().catch(() => {});
192}
193 
194async function isInteractionSessionReady(
195  context: AdapterContext,
196  interaction: WaitForInteractionRequest,
197): Promise<boolean> {
198  if (interaction.provider !== "x") {
199    return true;
200  }
201  return await isXSessionReady(context).catch(() => false);
202}
203 
204async function reopenInteractiveRuntime(
205  runtime: RuntimeResources,
206  options: ConvertCommandOptions,
207  debugEnabled: boolean,
208): Promise<RuntimeResources> {
209  if (runtime.interactive) {
210    return runtime;
211  }
212 
213  await closeRuntime(runtime);
214  return openRuntime(options, true, debugEnabled);
215}
216 
217async function captureForceWaitSnapshot(
218  adapter: Adapter,
219  context: AdapterContext,
220): Promise<ForceWaitSnapshot> {
221  const [gate, url, login] = await Promise.all([
222    detectInteractionGate(context.browser).catch(() => null),
223    context.browser.getURL().catch(() => context.input.url.toString()),
224    adapter.checkLogin?.(context).catch(() => ({
225      provider: adapter.name,
226      state: "unknown" as const,
227    })),
228  ]);
229 
230  return {
231    url,
232    hasGate: Boolean(gate),
233    loginState: login?.state ?? "unavailable",
234    sessionReady: adapter.name === "x" ? await isXSessionReady(context).catch(() => false) : true,
235  };
236}
237 
238async function waitForForceResume(
239  adapter: Adapter,
240  context: AdapterContext,
241  options: ConvertCommandOptions,
242): Promise<void> {
243  if (context.interactive) {
244    await context.browser.bringToFront().catch(() => {});
245  }
246 
247  const prompt =
248    "Chrome is ready. Complete any manual login or verification. Extraction will continue automatically after it detects progress, or press Enter to continue immediately.";
249  context.log.info(prompt);
250 
251  const rl = createInterface({
252    input: process.stdin,
253    output: process.stderr,
254  });
255 
256  let manualContinue = false;
257  let closed = false;
258  const closeReadline = (): void => {
259    if (!closed) {
260      closed = true;
261      rl.close();
262    }
263  };
264 
265  rl.once("line", () => {
266    manualContinue = true;
267    closeReadline();
268  });
269 
270  const initial = await captureForceWaitSnapshot(adapter, context);
271  const startedAt = Date.now();
272 
273  try {
274    while (Date.now() - startedAt < options.interactionTimeoutMs) {
275      if (manualContinue) {
276        return;
277      }
278 
279      const current = await captureForceWaitSnapshot(adapter, context);
280      if (shouldAutoContinueForceWait(initial, current)) {
281        return;
282      }
283 
284      await sleep(options.interactionPollIntervalMs);
285    }
286  } finally {
287    closeReadline();
288  }
289 
290  throw new Error("Timed out waiting for force-mode interaction to complete");
291}
292 
293async function waitForInteraction(
294  adapter: Adapter,
295  context: AdapterContext,
296  interaction: WaitForInteractionRequest,
297  options: ConvertCommandOptions,
298): Promise<AdapterLoginInfo> {
299  const timeoutMs = interaction.timeoutMs ?? options.interactionTimeoutMs;
300  const pollIntervalMs = interaction.pollIntervalMs ?? options.interactionPollIntervalMs;
301  if (context.interactive) {
302    await context.browser.bringToFront().catch(() => {});
303  }
304  context.log.info(interaction.prompt);
305 
306  const startedAt = Date.now();
307  let lastLogin: AdapterLoginInfo | null = null;
308 
309  while (Date.now() - startedAt < timeoutMs) {
310    if (interaction.kind === "login" && adapter.checkLogin) {
311      lastLogin = await adapter.checkLogin(context);
312      if (lastLogin.state === "logged_in" && await isInteractionSessionReady(context, interaction)) {
313        return lastLogin;
314      }
315    }
316 
317    const gate = await detectInteractionGate(context.browser);
318    if (!gate) {
319      if (interaction.kind !== "login") {
320        return lastLogin ?? {
321          provider: interaction.provider,
322          state: "unknown",
323          reason: `${interaction.provider} challenge cleared`,
324        };
325      }
326 
327      if (!adapter.checkLogin) {
328        return {
329          provider: interaction.provider,
330          state: "unknown",
331        };
332      }
333 
334      lastLogin = await adapter.checkLogin(context);
335      if (lastLogin.state !== "logged_out" && await isInteractionSessionReady(context, interaction)) {
336        return lastLogin;
337      }
338    }
339    await sleep(pollIntervalMs);
340  }
341 
342  const reason = lastLogin?.reason ? ` (${lastLogin.reason})` : "";
343  throw new Error(`Timed out waiting for ${interaction.provider} interaction${reason}`);
344}
345 
346export function formatOutputContent(
347  format: OutputFormat,
348  payload: SuccessfulConvertOutput | InteractionRequiredOutput,
349): string {
350  if (format === "json") {
351    return JSON.stringify(payload, null, 2);
352  }
353 
354  if (payload.status !== "ok") {
355    throw new Error("Markdown output is only available for successful extraction results");
356  }
357 
358  return payload.markdown;
359}
360 
361function printOutput(content: string): void {
362  process.stdout.write(content);
363  if (!content.endsWith("\n")) {
364    process.stdout.write("\n");
365  }
366}
367 
368export async function runConvertCommand(options: ConvertCommandOptions): Promise<void> {
369  if (!options.url) {
370    throw new Error("URL is required");
371  }
372  if (options.downloadMedia && !options.output) {
373    throw new Error("--download-media requires --output so media paths can be rewritten relative to the saved output file");
374  }
375 
376  const url = normalizeUrl(options.url);
377  let runtime = await openRuntime(options, options.waitMode !== "none", Boolean(options.debugDir));
378  const logger = createLogger(Boolean(options.debugDir));
379  let didLogin = false;
380  let adapter: Adapter | null = null;
381  let context: AdapterContext | null = null;
382 
383  try {
384    adapter = resolveAdapter({ url }, options.adapter);
385    context = {
386      input: { url },
387      browser: runtime.browser,
388      network: runtime.network,
389      cdp: runtime.cdp,
390      log: logger,
391      outputFormat: options.format,
392      timeoutMs: options.timeoutMs,
393      interactive: runtime.interactive,
394      downloadMedia: options.downloadMedia,
395    };
396 
397    if (adapter.restoreCookies) {
398      const restored = await adapter.restoreCookies(context, runtime.chrome.profileDir).catch(() => false);
399      if (restored) logger.info(`Restored ${adapter.name} session cookies from sidecar.`);
400    }
401 
402    if (options.waitMode === "interaction" && adapter.checkLogin) {
403      await context.browser.goto(url.toString(), options.timeoutMs).catch(() => {});
404      const preLogin = await adapter.checkLogin(context);
405      if (preLogin.state !== "logged_in") {
406        didLogin = true;
407        await waitForInteraction(adapter, context, {
408          type: "wait_for_interaction",
409          kind: "login",
410          provider: preLogin.provider ?? adapter.name,
411          prompt: `Please sign in to ${adapter.name === "x" ? "X" : adapter.name} in the opened Chrome window. Extraction will continue automatically once login is detected.`,
412          reason: preLogin.reason ?? `Not logged in to ${adapter.name}`,
413          requiresVisibleBrowser: true,
414        }, options);
415      }
416    }
417 
418    if (options.waitMode === "force") {
419      await context.browser.goto(url.toString(), options.timeoutMs).catch(() => {});
420      await waitForForceResume(adapter, context, options);
421    }
422 
423    let result = await adapter.process(context);
424 
425    if (result.status === "no_document") {
426      const interaction = await detectInteractionGate(context.browser);
427      if (interaction) {
428        result = {
429          status: "needs_interaction",
430          interaction,
431          login: result.login,
432        };
433      }
434    }
435 
436    while (result.status === "needs_interaction") {
437      if (options.waitMode === "none") {
438        if (options.format === "json") {
439          printOutput(
440            formatOutputContent(options.format, {
441            adapter: adapter.name,
442            status: result.status,
443            login: result.login,
444            interaction: result.interaction,
445            }),
446          );
447          return;
448        }
449 
450        throw new Error(`${adapter.name} requires manual interaction. Re-run with --wait-for interaction to continue after completing it.`);
451      }
452 
453      if (result.interaction.requiresVisibleBrowser !== false) {
454        runtime = await reopenInteractiveRuntime(runtime, options, Boolean(options.debugDir));
455      }
456 
457      context = {
458        input: { url },
459        browser: runtime.browser,
460        network: runtime.network,
461        cdp: runtime.cdp,
462        log: logger,
463        outputFormat: options.format,
464        timeoutMs: options.timeoutMs,
465        interactive: runtime.interactive,
466        downloadMedia: options.downloadMedia,
467      };
468 
469      await context.browser.goto(url.toString(), options.timeoutMs).catch(() => {});
470      if (result.interaction.kind === "login") {
471        didLogin = true;
472      }
473      await waitForInteraction(adapter, context, result.interaction, options);
474      result = await adapter.process(context);
475 
476      if (result.status === "no_document") {
477        const interaction = await detectInteractionGate(context.browser);
478        if (interaction) {
479          result = {
480            status: "needs_interaction",
481            interaction,
482            login: result.login,
483          };
484        }
485      }
486    }
487 
488    let document: ExtractedDocument | null = result.status === "ok" ? result.document : null;
489    let media: MediaAsset[] = result.status === "ok" ? (result.media ?? []) : [];
490    let login = result.login;
491    let mediaAdapter = adapter;
492 
493    if (!document && adapter.name !== genericAdapter.name && result.status === "no_document") {
494      logger.info(`Adapter ${adapter.name} returned no structured document; falling back to generic extraction`);
495      const fallback = await genericAdapter.process(context);
496      if (fallback.status === "ok") {
497        document = fallback.document;
498        media = fallback.media ?? [];
499        mediaAdapter = genericAdapter;
500      }
501    }
502 
503    if (!document) {
504      throw new Error("Failed to extract a document from the target URL");
505    }
506 
507    document.requestedUrl ??= url.toString();
508 
509    let markdown = renderMarkdown(document);
510    let downloadResult:
511      | Awaited<ReturnType<typeof downloadMediaAssets>>
512      | null = null;
513 
514    if (options.downloadMedia && options.output) {
515      downloadResult = mediaAdapter.downloadMedia
516        ? await mediaAdapter.downloadMedia({
517            media,
518            outputPath: options.output,
519            mediaDir: options.mediaDir,
520            log: logger,
521          })
522        : await downloadMediaAssets({
523            media,
524            outputPath: options.output,
525            mediaDir: options.mediaDir,
526            log: logger,
527          });
528 
529      markdown = rewriteMarkdownMediaLinks(markdown, downloadResult.replacements);
530      if (downloadResult.downloadedImages > 0 || downloadResult.downloadedVideos > 0) {
531        logger.info(
532          `Downloaded ${downloadResult.downloadedImages} images and ${downloadResult.downloadedVideos} videos`,
533        );
534      }
535    }
536 
537    if (options.output) {
538      await writeOutput(
539        options.output,
540        formatOutputContent(options.format, {
541          adapter: document.adapter ?? adapter.name,
542          status: "ok",
543          login,
544          media,
545          downloads: downloadResult,
546          document,
547          markdown,
548        }),
549      );
550      logger.info(`Saved ${options.format} to ${options.output}`);
551    }
552 
553    if (options.debugDir) {
554      await writeDebugArtifacts(options.debugDir, document, markdown, runtime.browser, runtime.network);
555      logger.info(`Wrote debug artifacts to ${options.debugDir}`);
556    }
557 
558    if (options.format === "json") {
559      printOutput(
560        formatOutputContent(options.format, {
561        adapter: document.adapter ?? adapter.name,
562        status: "ok",
563        login,
564        media,
565        downloads: downloadResult,
566        document,
567        markdown,
568        }),
569      );
570      return;
571    }
572 
573    printOutput(markdown);
574  } finally {
575    if (adapter?.exportCookies && context) {
576      await adapter.exportCookies(context, runtime.chrome.profileDir).catch(() => {});
577    }
578    await closeRuntime(runtime);
579  }
580}
581