fetchClosure

tcmts.cli.sync.fetchClosure · #p7rbap3m · term · runs on the server

uses node:fs

Fetch the components needed for hashes that we lack, following dependencies until the closure is complete. What's fetched is set aside in a file as it comes, only what orders it kept in memory: a long history's definitions, IR and all, wouldn't fit. read gets one back; dispose removes the file.

import type { Codebase } from "#codebase/tcmts/cli/codebase";
import { chunks } from "#codebase/tcmts/cli/sync";
import type { FetchedRecord, Remote } from "#codebase/tcmts/cli/sync";
import { shortHash } from "#codebase/tcmts/core/hash";
import type { DefRecord } from "#codebase/tcmts/core/types";
import fs from "node:fs";
import os from "node:os";
import path_1 from "node:path";

export async function fetchClosure(
  cb: Codebase,
  remote: Remote,
  hashes: string[],
): Promise<{
  records: FetchedRecord[];
  read: (r: FetchedRecord) => DefRecord;
  dispose: () => void;
}> {
  const dir = fs.mkdtempSync(path_1.join(os.tmpdir(), "tcm-fetch-"));
  const fd = fs.openSync(path_1.join(dir, "records.jsonl"), "w+");
  // gone from the directory at once, where the system allows it (the file lives on until it's closed), so a pull that
  // ends some other way (killed, out of memory) leaves nothing behind; elsewhere `dispose` removes it
  try {
    fs.rmSync(dir, { recursive: true });
  } catch {
    // (Windows: an open file stays)
  }
  const dispose = () => {
    fs.closeSync(fd);
    fs.rmSync(dir, { recursive: true, force: true });
  };
  let end = 0;
  const keep = (r: DefRecord): FetchedRecord => {
    const bytes = Buffer.from(JSON.stringify(r));
    fs.writeSync(fd, bytes, 0, bytes.length, end);
    const kept = {
      hash: r.hash,
      componentHash: r.componentHash,
      deps: r.deps,
      at: end,
      size: bytes.length,
    };
    end += bytes.length;
    return kept;
  };
  const got = new Map<string, FetchedRecord>();
  // (a definition can come again with another text: its comments, layout or doc, which `receive` keeps beside it)
  const texts: FetchedRecord[] = [];
  try {
    let want = [...new Set(hashes)].filter((h) => !cb.store.hasDef(h));
    while (want.length) {
      const fresh: FetchedRecord[] = [];
      for (const c of chunks(want))
        for (const r of await remote.get(c)) {
          if (got.has(r.hash)) texts.push(keep(r));
          else {
            const kept = keep(r);
            got.set(r.hash, kept);
            fresh.push(kept);
          }
        }
      const absent = want.filter((h) => !got.has(h));
      if (absent.length)
        throw new Error(
          `${remote.label} does not have ${absent.map((h) => shortHash(h)).join(", ")}`,
        );
      // what the new ones use (what came before was followed already)
      want = [
        ...new Set(
          fresh.flatMap((r) =>
            r.deps.filter((d) => d.ref.startsWith("#")).map((d) => d.ref.slice(1)),
          ),
        ),
      ].filter((h) => !cb.store.hasDef(h) && !got.has(h));
    }
  } catch (e) {
    dispose();
    throw e;
  }
  return {
    records: [...got.values(), ...texts],
    read: (r) => {
      const bytes = Buffer.alloc(r.size);
      fs.readSync(fd, bytes, 0, r.size, r.at);
      return JSON.parse(bytes.toString("utf8")) as DefRecord;
    },
    dispose,
  };
}

uses

used by