crawl

tcmts.cli.test.e2e.web.crawl · #chm36j43 · term · runs anywhere

Every module a browser would load for the app, following imports from the entry. While Vite's dependency optimizer is still scanning it can answer with a non-200 (a real browser recovers via Vite's client), so a crawl that meets one starts over.

import { get } from "#codebase/tcmts/cli/test/e2e/web";

export async function crawl(url: string, attempts = 6): Promise<Map<string, string>> {
  const seen = new Map<string, string>();
  const queue = [new URL("/entry.ts", url).href];
  while (queue.length) {
    const u = queue.pop()!;
    if (seen.has(u)) continue;
    const res = await get(u);
    if (res.status !== 200) {
      if (attempts <= 1) throw new Error(`${u}: ${res.status}`);
      await new Promise((r) => setTimeout(r, 500));
      return crawl(url, attempts - 1);
    }
    const text = await res.text();
    seen.set(u, text);
    for (const m of text.matchAll(/(?:\bfrom\s*|\bimport\s*\(?\s*)["']([^"']+)["']/g)) if (/^[./]/.test(m[1]!)) queue.push(new URL(m[1]!, u).href);
  }
  return seen;
}

uses

used by