buildCodeBrowser

tcmts.dev.buildCodeBrowser · #rubllwlg · term · runs on the server

uses node:path

The docs site's code browser (code/): tcm's own codebase, and each example as the codebase its setup script builds (kept in dist/examples by what its files hash to, so it's built again only when they change, and kept a day after it was last used, as verifies of other branches may have other files: each built one at a time); and each example's files to download (examples/<name>.tar.gz, a directory of its name). Returns what couldn't be built, and any link in it to a page that isn't there: named exactly, case and all (a filesystem that ignores case would find Highlighter.html for highlighter.html; the server's doesn't). A large example (Effect: minutes to import) is built too only with large (deploys, pnpm docs:site); otherwise it's only its files to download. env is given to the examples' setup and builds: a slot's (withSlot), when this runs in one, so they don't wait for another. base is the path the site is served under: an example that's a static site is built for its place in it.

import { IR_VERSION } from "#codebase/tcmts/core/hash";
import { codebaseDir, EXAMPLES, holdLock, lockHeld, repository, sh, shippedLibrariesDir } from "#codebase/tcmts/dev";
import { esc, writeCodeBrowser } from "#codebase/tcmts/site";
import type { Highlighter } from "#codebase/tcmts/site";
import { createHash } from "node:crypto";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";

export async function buildCodeBrowser(
  out: string,
  hl: Highlighter,
  large = false,
  env: Record<string, string> = {},
  base = "",
): Promise<string[]> {
  const tcm = process.env.TCM_BIN;
  if (!tcm)
    return [
      "the code browser is built with `tcm run`, which gives it the codebase's own tcm (TCM_BIN)",
    ];
  const problems: string[] = [];
  const browsable: { name: string; title: string; count: number; run?: string }[] = [];
  const count = writeCodeBrowser(
    { name: "tcm", title: "tcm-ts", dir: codebaseDir(), root: "tcmts" },
    path.join(out, "code", "tcm"),
    hl,
  );
  browsable.push({
    name: "tcm",
    title: "tcm itself: the codebase manager, its editor extension, these docs",
    count,
  });
  fs.mkdirSync(path.join(out, "examples"), { recursive: true });
  // (the libraries tcm ships, which their setups mount: found while this is still the codebase's branch)
  const libraries = shippedLibrariesDir();
  // (the examples' codebases are their own, not on the branch of a session verifying tcm's: put back below)
  const session = process.env.TCM_BRANCH;
  delete process.env.TCM_BRANCH;
  for (const ex of EXAMPLES) {
    const files = fs.mkdtempSync(path.join(os.tmpdir(), `tcm example ${ex.name} `));
    for (const [rel, text] of Object.entries(ex.files)) {
      const file = path.join(files, ex.name, rel);
      fs.mkdirSync(path.dirname(file), { recursive: true });
      fs.writeFileSync(file, text, { mode: rel.endsWith(".sh") ? 0o755 : 0o644 });
    }
    // (and by the hash format: a codebase in another one can't be read, so it's built again)
    const key = createHash("sha256")
      .update(JSON.stringify([IR_VERSION, ex.files]))
      .digest("hex")
      .slice(0, 16);
    const built = path.join(repository(), "dist", "examples", `${ex.name}-${key}`);
    // its download, and its pages, kept beside the codebase they're made from: the same files make the same ones
    const archive = `${built}.tar.gz`;
    // (verifies of other branches build these too, side by side: one at a time for each)
    const done = await holdLock(`${built}.lock`, `another verify building examples/${ex.name}`);
    if (!fs.existsSync(archive)) {
      const part = `${archive}.part-${process.pid}`;
      const tar = sh("tar", ["-czf", part, "-C", files, ex.name], files);
      if (tar.ok) fs.renameSync(part, archive);
      else problems.push(`examples/${ex.name}.tar.gz: ${tar.out.trim()}`);
    }
    if (fs.existsSync(archive))
      fs.copyFileSync(archive, path.join(out, "examples", `${ex.name}.tar.gz`));
    // a large one, unless asked for, is only its files to download: building it takes minutes
    if (ex.large && !large) {
      done();
      continue;
    }
    if (!fs.existsSync(path.join(built, ".tcm", "codebase.sqlite"))) {
      fs.rmSync(built, { recursive: true, force: true });
      // (its tcm works on the example's codebase, not the one running this; its tests are the suites' to run, with
      // tcm as it's packed: `tcmts.cli.test.e2e.examples`)
      const r = sh(path.join(files, ex.name, "setup.sh"), [built], files, {
        ...env,
        TCM_BIN: tcm,
        TCM_CODEBASE: "",
        TCM_SHIPPED_LIBRARIES: libraries,
        NO_TESTS: "1",
      });
      if (!r.ok) {
        // (not kept half made: the next build runs its setup again)
        fs.rmSync(built, { recursive: true, force: true });
        problems.push(
          `examples/${ex.name}: setup.sh failed:\n${r.out.trim().split("\n").slice(-8).join("\n")}`,
        );
        done();
        continue;
      }
    }
    // (and by the code writing them, whose hash covers all it uses; without one, run from a scratch file, they're
    // written each time)
    const writer = (writeCodeBrowser as unknown as Record<symbol, string | undefined>)[
      Symbol.for("tcm.hash")
    ];
    const pages = writer && `${built}-pages-${writer.slice(0, 16)}`;
    const to = path.join(out, "code", ex.name);
    let count: number;
    if (pages && fs.existsSync(path.join(pages, "count"))) {
      count = Number(fs.readFileSync(path.join(pages, "count"), "utf8"));
      fs.cpSync(path.join(pages, "code"), to, { recursive: true });
    } else {
      count = writeCodeBrowser({ name: ex.name, title: ex.name, dir: built, root: "" }, to, hl);
      if (pages) {
        fs.rmSync(pages, { recursive: true, force: true });
        fs.cpSync(to, path.join(pages, "code"), { recursive: true });
        fs.writeFileSync(path.join(pages, "count"), String(count));
      }
    }
    // one that's a static site is built into this one, under its path
    if (ex.site) {
      const r = sh(
        process.execPath,
        [
          tcm,
          "build",
          ex.site,
          "--static",
          path.join(out, "examples", ex.name),
          "--base",
          `${base}/examples/${ex.name}`,
        ],
        built,
        { ...env, TCM_CODEBASE: "" },
      );
      if (!r.ok)
        problems.push(
          `examples/${ex.name}: tcm build --static failed:\n${r.out.trim().split("\n").slice(-8).join("\n")}`,
        );
    }
    browsable.push({
      name: ex.name,
      title: ex.title,
      count,
      ...(ex.app || ex.site ? { run: `../examples/${ex.name}/` } : {}),
    });
    // what earlier files or an earlier writer made isn't kept: once it's gone unused a day (another branch's verify
    // may use it), and not while it's being built
    const now = new Date();
    const kept = [built, archive, ...(pages ? [pages] : [])].filter((p) => fs.existsSync(p));
    for (const p of kept) fs.utimesSync(p, now, now);
    done();
    const examples = path.dirname(built);
    for (const e of fs.readdirSync(examples)) {
      const of = new RegExp(`^${ex.name}-([0-9a-f]{16})(?![^.-])`).exec(e);
      if (!of || kept.some((p) => path.basename(p) === e) || e.endsWith(".lock")) continue;
      if (lockHeld(path.join(examples, `${ex.name}-${of[1]}.lock`))) continue;
      let unused = Infinity;
      try {
        unused = Date.now() - fs.statSync(path.join(examples, e)).mtimeMs;
      } catch {}
      if (unused > 86_400_000) fs.rmSync(path.join(examples, e), { recursive: true, force: true });
    }
  }
  if (session !== undefined) process.env.TCM_BRANCH = session;
  const list = browsable
    .map(
      (b) =>
        `<dt><a href="${b.name}/index.html">${esc(b.name)}</a>${b.run ? ` · <a href="${b.run}">run it</a>` : ""}</dt><dd>${esc(b.title)} <span class="sig">(${b.count} definitions)</span></dd>`,
    )
    .join("\n");
  fs.writeFileSync(
    path.join(out, "code", "index.html"),
    `<!doctype html>\n<html lang="en"><head>\n<meta charset="utf-8">\n<meta name="viewport" content="width=device-width, initial-scale=1">\n<title>code — tcm-ts</title>\n<link rel="stylesheet" href="../style.css">\n</head><body>\n<div class="site">\n<nav class="side" aria-label="codebases">\n<a class="home" href="../index.html">tcm-ts.txt</a>\n<div class="rule" aria-hidden="true"></div>\n</nav>\n<main>\n<h1>Code</h1>\n<div class="rule" aria-hidden="true"></div>\n<p>Codebases to browse, as tcm keeps them: every definition with its doc, its code with each reference a link, where it runs, and what it uses and what uses it. The examples are codebases their setup scripts build; download one to build it yourself: <code>curl -fsSL &lt;this site&gt;/examples/polls.tar.gz | tar xz &amp;&amp; polls/setup.sh /tmp/polls</code>.</p>\n<dl>\n${list}\n</dl>\n</main>\n</div>\n</body></html>\n`,
  );
  // the apps among them, for tcm's server to run (`tcm start <app> --base /examples/<name> <flags>`): a line each,
  // `<name> <app> <reset: yes|no> <flags>`
  const apps = EXAMPLES.filter((e) => e.app);
  fs.writeFileSync(
    path.join(out, "examples", "apps.txt"),
    apps
      .map(
        (e) =>
          [e.name, e.app!.name, e.app!.reset ? "yes" : "no", ...(e.app!.flags ?? [])].join(" ") +
          "\n",
      )
      .join(""),
  );
  // (where they run is the server's, not a file of the site)
  const running = new Set(apps.map((e) => `examples/${e.name}/`));
  const files = new Set<string>();
  const walk = (dir: string) => {
    for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
      const p = path.join(dir, e.name);
      if (e.isDirectory()) walk(p);
      else files.add(path.relative(out, p).split(path.sep).join("/"));
    }
  };
  walk(out);
  for (const f of [...files].filter((f) => f.startsWith("code/") && f.endsWith(".html"))) {
    const html = fs.readFileSync(path.join(out, f), "utf8");
    for (const [, to] of html.matchAll(/href="([^"#:]+)(?:#[^"]*)?"/g)) {
      const target = path.posix.normalize(path.posix.join(path.posix.dirname(f), to!));
      // (a directory is its index.html: a static site's, built in)
      if (
        !files.has(target) &&
        !files.has(path.posix.join(target, "index.html")) &&
        !running.has(target)
      )
        problems.push(`${f}: link to ${to}: no such page`);
    }
  }
  return problems;
}

uses

used by