componentBatches

tcmts.cli.sync.componentBatches · #7ezjkuzp · term · runs on the server

uses Buffer

records in batches a receiver can check one after another: whole components, each after the components among them that it uses (the receiver checks every reference resolves), and each batch under a request's size limit (a server takes 64 MB; the fewer batches, the less each checker compiles again what they all use) (sizeOf says a record's), however many there are. Further records of a definition (its other texts) come after all the code.

import type { DefRecord } from "#codebase/tcmts/core/types";

export function componentBatches<R extends Pick<DefRecord, "hash" | "componentHash" | "deps">>(
  records: R[],
  maxBytes = 40 * 1024 * 1024,
  maxRecords = 8000,
  sizeOf: (r: R) => number = (r) => Buffer.byteLength(JSON.stringify(r)),
): R[][] {
  const byComp = new Map<string, R[]>();
  const compOf = new Map<string, string>();
  const texts: R[] = [];
  for (const r of records) {
    if (compOf.has(r.hash)) {
      texts.push(r);
      continue;
    }
    compOf.set(r.hash, r.componentHash);
    byComp.set(r.componentHash, [...(byComp.get(r.componentHash) ?? []), r]);
  }
  // depth first, what a component uses before it (components hold any cycle, so the graph they make has none);
  // iteratively, so a long chain of versions can't overflow the stack
  const ordered: string[] = [];
  const placed = new Set<string>();
  for (const root of byComp.keys()) {
    const stack: [string, boolean][] = [[root, false]];
    while (stack.length) {
      const [c, done] = stack.pop()!;
      if (done) {
        ordered.push(c);
        continue;
      }
      if (placed.has(c)) continue;
      placed.add(c);
      stack.push([c, true]);
      for (const m of byComp.get(c)!)
        for (const d of m.deps) {
          const dep = d.ref.startsWith("#") ? compOf.get(d.ref.slice(1)) : undefined;
          if (dep && dep !== c && !placed.has(dep)) stack.push([dep, false]);
        }
    }
  }
  const batches: R[][] = [];
  let batch: R[] = [];
  let size = 0;
  const add = (rs: R[]) => {
    const bytes = rs.reduce((n, r) => n + sizeOf(r), 0);
    if (batch.length && (size + bytes > maxBytes || batch.length >= maxRecords)) {
      batches.push(batch);
      batch = [];
      size = 0;
    }
    batch.push(...rs);
    size += bytes;
  };
  for (const c of ordered) add(byComp.get(c)!);
  for (const t of texts) add([t]);
  if (batch.length) batches.push(batch);
  return batches;
}

uses

used by