fetchNamespaces

tcmts.cli.sync.fetchNamespaces · #avh7lgj7 · term · runs on the server

uses tcmts.cli.sync.componentBatches, which uses Buffer

Fetch what this codebase lacks of the remote's namespace nodes from roots down, with the definitions they name (and what those use), the text versions they show, and the package versions their code runs on: everything to read those namespaces here. Returns definitions received. maxNames refuses namespaces naming more definitions than that before any are fetched (a hub mirroring a stranger's release, say); keepLocks keeps the package versions this codebase has locked (a library's code doesn't move them).

import type { Codebase } from "#codebase/tcmts/cli/codebase";
import { chunks, componentBatches, fetchClosure } from "#codebase/tcmts/cli/sync";
import type { NamespaceNode, Remote } from "#codebase/tcmts/cli/sync";
import { UsageError } from "#codebase/tcmts/cli/ui";
import type { NamespaceBody } from "#codebase/tcmts/store";

export async function fetchNamespaces(
  cb: Codebase,
  remote: Remote,
  roots: string[],
  o: { maxNames?: number; keepLocks?: boolean } = {},
): Promise<number> {
  const nodes: NamespaceNode[] = [];
  const asked = new Set<string>();
  const named = new Set<string>();
  for (let level = [...new Set(roots)]; level.length;) {
    const missing = cb.store.missingNamespaces(level).filter((h) => !asked.has(h));
    for (const h of missing) asked.add(h);
    const got: NamespaceNode[] = [];
    for (const c of chunks(missing)) got.push(...(await remote.getNamespaces(c)));
    if (got.length !== missing.length)
      throw new Error(`${remote.label} is missing part of its history`);
    // (only what was asked for: hash-checked as it's stored, but nothing else is taken)
    const wanted = new Set(missing);
    if (got.some((n) => !wanted.has(n.hash)))
      throw new Error(`${remote.label} sent namespaces it wasn't asked for`);
    nodes.push(...got);
    const bodies = got.map(
      (n) => JSON.parse(n.body) as ["ns", [string, string[]][], [string, string][]],
    );
    for (const b of bodies) for (const [, hs] of b[1]) for (const d of hs) named.add(d);
    level = [...new Set(bodies.flatMap((b) => b[2].map(([, child]) => child)))];
    if (o.maxNames !== undefined && named.size > o.maxNames)
      throw new UsageError(
        `${remote.label}'s namespace names more than ${o.maxNames} definitions: more than is fetched from it`,
      );
  }
  const fetched = await fetchClosure(cb, remote, [...named]);
  let stored = 0;
  try {
    // the versions their code runs on, then the code (verified on them), then the namespaces naming it
    if (fetched.records.length)
      cb.adoptPackages(cb.packagesUsedBy(fetched.records, await remote.packages()), {
        keepLocks: o.keepLocks,
      });
    // (a batch at a time, as a push sends them, read back as each is checked: a long history wouldn't fit in memory)
    for (const batch of componentBatches(fetched.records, undefined, undefined, (r) => r.size))
      stored += (await cb.receiveInParallel(batch.map(fetched.read))).stored;
  } finally {
    fetched.dispose();
  }
  // and the text versions their names show (comments, layout, doc) that this codebase lacks, of code it has now
  const unread = new Set<string>();
  for (const n of nodes)
    for (const [, d, t] of (JSON.parse(n.body) as NamespaceBody)[3] ?? [])
      if (!cb.store.textVersion(d, t)) unread.add(d);
  for (const c of chunks([...unread])) await cb.receiveInParallel(await remote.get(c));
  cb.store.putNamespaces(nodes);
  return stored;
}

uses

used by