materialize

tcmts.loader.materialize · #chldxjva · term · runs on the server

uses node:path

Write the cache modules hashes need: their components and everything those import. keepExisting leaves a module that's already there as it is, rather than rendering it again: a component's code never changes, only the names and source maps it's rendered with, which is all an internal re-check needs. trustExisting goes further: an existing module's imports aren't followed either (they were written with it), so only what's new is visited; for the #codebase shim after a change, which needs modules to import, not their names up to date.

Otherwise a module is rendered again only when the names it's rendered with (its members' and what they refer to), its members' docs (its source maps point into views, which show them), or the views it points into (each branch has its own: src.views) changed since it was: those, and its code, are all that's in it. (What each was rendered with is in rendered.json, read once: a closure has hundreds of modules. Two processes rendering at once can lose each other's entries, which only renders those again.)

import { extRefString } from "#codebase/tcmts/core/types";
import type { DefRecord, ExtRef } from "#codebase/tcmts/core/types";
import { fileModule, moduleFile, render, writeIfChanged } from "#codebase/tcmts/loader";
import type { DefSource, Layout } from "#codebase/tcmts/loader";
import { ensureExternals } from "#codebase/tcmts/loader/packages";
import { createHash } from "node:crypto";
import fs from "node:fs";
import path_2 from "node:path";

export function materialize(
  hashes: Iterable<string>,
  src: DefSource,
  lay: Layout,
  opts: { keepExisting?: boolean; trustExisting?: boolean; also?: ExtRef[] } = {},
): void {
  const seen = new Set<string>();
  const exts = new Map<string, ExtRef>();
  const stamps = path_2.join(lay.cacheDir, "rendered.json");
  let rendered: Record<string, string> = {};
  try {
    rendered = JSON.parse(fs.readFileSync(stamps, "utf8")) as Record<string, string>;
  } catch {
    // (none yet, or cut short: everything is rendered again)
  }
  let changed = false;
  const renderer =
    (render as unknown as Record<symbol, string | undefined>)[Symbol.for("tcm.hash")] ?? "";
  const queue = [...hashes];
  // (closures share most of what they refer to: each name is looked up once)
  const nameMemo = new Map<string, string>();
  const nameOf = (x: string) => {
    let n = nameMemo.get(x);
    if (n === undefined) nameMemo.set(x, (n = src.nameOf(x) ?? ""));
    return n;
  };
  const kept = (x: string) => src.kept?.(x) ?? undefined;
  const undecided: string[] = [];
  /** render a component's module if what it's rendered with changed: whether it was */
  const write = (comp: string, members: DefRecord[], file: string): boolean => {
    const named = [
      ...new Set(
        members.flatMap((m) => [m.hash, ...m.spans.flatMap((s) => (s.hash ? [s.hash] : []))]),
      ),
    ].map((x) => `${x}=${nameOf(x)}`);
    const names = createHash("sha256")
      .update(
        JSON.stringify([
          named,
          members.map((m) => m.doc ?? ""),
          members.map((m) => kept(m.hash)?.value ?? ""),
          src.views ?? "",
          // and how modules are rendered: a tcm that renders them otherwise renders each again, once
          renderer,
        ]),
      )
      .digest("hex")
      .slice(0, 32);
    if (rendered[comp] === names && fs.existsSync(file)) return false;
    writeIfChanged(file, render(members, src, lay, { style: "cache", kept }).text);
    rendered[comp] = names;
    return true;
  };
  while (queue.length) {
    const h = queue.pop()!;
    const rec = src.get(h);
    if (!rec) throw new Error(`missing definition #${h}`);
    if (seen.has(rec.componentHash)) continue;
    seen.add(rec.componentHash);
    if (
      opts.trustExisting &&
      fs.existsSync(path_2.join(lay.cacheDir, moduleFile(rec.componentHash, src)))
    )
      continue;
    const members = src.members(rec.componentHash);
    // a file kept as written is a value: its content, as saved (a string, or bytes)
    if (rec.kind === "file") {
      const file = path_2.join(lay.cacheDir, moduleFile(rec.componentHash, src));
      if (!(opts.keepExisting && fs.existsSync(file))) writeIfChanged(file, fileModule(rec));
      continue;
    }
    for (const m of members) {
      for (const s of m.spans) {
        if (s.hash) queue.push(s.hash);
        else if (s.ext) exts.set(extRefString(s.ext), s.ext);
      }
      if (m.jsx) exts.set(m.jsx, { module: m.jsx, subpath: "jsx-runtime", export: "*" });
    }
    const file = path_2.join(lay.cacheDir, moduleFile(rec.componentHash, src));
    if (opts.keepExisting && fs.existsSync(file)) continue;
    const ask = members.filter((m) => src.kept?.(m.hash) === undefined);
    if (src.keep && ask.length) undecided.push(...ask.map((m) => m.hash));
    changed = write(rec.componentHash, members, file) || changed;
  }
  // pure constants not evaluated yet: evaluated now, together (their modules are written), and their modules
  // written again with the values kept
  if (undecided.length && src.keep) {
    src.keep(undecided);
    for (const h of undecided) {
      const rec = src.get(h)!;
      if (!src.kept?.(h)) continue;
      changed =
        write(
          rec.componentHash,
          src.members(rec.componentHash),
          path_2.join(lay.cacheDir, moduleFile(rec.componentHash, src)),
        ) || changed;
    }
  }
  if (changed) {
    // (whole or not at all: written beside, then moved over)
    const part = `${stamps}.${process.pid}`;
    fs.mkdirSync(lay.cacheDir, { recursive: true });
    fs.writeFileSync(part, JSON.stringify(rendered));
    fs.renameSync(part, stamps);
  }
  // (with the packages the caller needs beside these, a test runner's say: one install for all of them)
  ensureExternals(lay, [...exts.values(), ...(opts.also ?? [])]);
}

uses

used by