runSuites

tcmts.cli.suites.runSuites · #nwf5emd4 · term · runs on the server

uses process

Run suites against the codebase's own build of their commands (or builds, by command), reusing results. A test's result holds until it changes, or the code of a definition it executed does: one it didn't run can't change what it does without one it ran changing too. Each suite keeps its passing runs for several versions of that code (KEPT_RUNS), so a result holds again when the code it ran is back as it was (another branch's tests having run between). A test that executed nothing of the codebase always runs, and so does one that failed: a program can fail for what's around it (its environment, a machine's state), not its code, and a failure kept would outlive that. fresh runs them all.

import { inSuiteSlot, machineSlots, slotPool, slotsDir, slotWaits } from "#codebase/tcmts/cli/budget";
import type { SlotWaits } from "#codebase/tcmts/cli/budget";
import type { Codebase } from "#codebase/tcmts/cli/codebase";
import { dim } from "#codebase/tcmts/cli/style";
import {
  buildModules, codeDigest, KEPT_RUN_FORMAT, KEPT_RUNS, memberCode, namesByHash, ownCode, packRuns, runSuite,
  suiteApps, suiteLibraries, unpackRuns,
} from "#codebase/tcmts/cli/suites";
import type { Suite, SuiteRun } from "#codebase/tcmts/cli/suites";
import type { TestOutcome } from "#codebase/tcmts/cli/testing";
import { resultKey } from "#codebase/tcmts/cli/vitest";
import { writeRunner } from "#codebase/tcmts/loader";

export async function runSuites(
  cb: Codebase,
  suites: Suite[],
  fresh: boolean,
  onOutcome?: (o: TestOutcome) => void,
  builds = new Map<string, string>(),
  waits: SlotWaits = slotWaits((l) => console.error(dim(l))),
): Promise<TestOutcome[]> {
  // one result for the code a test ran, whichever build of the command ran it (the codebase's own, or a package
  // packed from it: the same definitions, by hash), so a test runs once for that code: a merge's run serves the
  // verify after it
  const keyOf = (_s: Suite) => resultKey(cb);
  // the names of what runs, as they are before it does (a save meanwhile replaces what a name points at, not what ran)
  const names = namesByHash(cb);
  const runs = unpackRuns(cb.store.getSetting("suites"));
  const slot = (s: Suite) => s.path;
  // suites run much of the same code: each name's own code is worked out once
  const owns = new Map<string, string | undefined>();
  const ownAt = (name: string) => {
    if (!owns.has(name)) {
      // (`Class#method`: a class's method a run used, or `Class#` the rest of it: `memberCode`)
      const at = name.indexOf("#");
      const hs = cb.store
        .hashesAt(at < 0 ? name : name.slice(0, at))
        .filter((h) => cb.store.getDef(h)?.kind !== "type");
      const rec = hs.length === 1 ? cb.store.getDef(hs[0]!)! : undefined;
      owns.set(name, rec && (at < 0 ? ownCode(cb, rec) : memberCode(cb, rec, name.slice(at + 1))));
    }
    return owns.get(name);
  };
  // against another build of a command, a result holds if the build has the code: every component of the command's
  // the test ran (an older tcm.js has other code, and its test runs)
  const builtHas = new Map([...builds].map(([cmd, file]) => [cmd, buildModules(file)]));
  const commandCode = new Map<string, Set<string>>();
  const inBuild = (s: Suite, name: string) => {
    const has = builtHas.get(s.bin);
    if (!has) return true;
    let code = commandCode.get(s.main);
    if (!code) {
      const h = cb.store.hashesAt(s.main).find((x) => cb.store.getDef(x)?.kind !== "type");
      commandCode.set(s.main, (code = new Set(h ? cb.closureComponents([h]) : [])));
    }
    const at = name.indexOf("#");
    const c = cb.store
      .hashesAt(at < 0 ? name : name.slice(0, at))
      .map((h) => cb.store.getDef(h)?.componentHash)[0];
    return !c || !code.has(c) || has.has(c);
  };
  const holds = (s: Suite, r: SuiteRun, of: string) => {
    if (!r.result.ok || r.of !== of || r.key !== keyOf(s) || r.ran.length === 0) return false;
    const codes = r.ran.map(ownAt);
    return (
      codes.every((c) => c !== undefined) &&
      codeDigest(codes as string[]) === r.code &&
      r.ran.every((n) => inBuild(s, n))
    );
  };
  const entries = new Map<string, string>();
  const entry = (s: Suite) => {
    const built = builds.get(s.bin);
    if (built) return built;
    // the apps its package ships, which a packed command has beside it
    const apps = suiteApps(cb, s.pkg);
    if (apps) process.env.TCM_APPS = apps;
    // and the libraries
    const libraries = suiteLibraries(cb, s.pkg);
    if (libraries) process.env.TCM_SHIPPED_LIBRARIES = libraries;
    const main = s.main;
    if (!entries.has(main)) {
      const h = cb.store.hashesAt(main).find((x) => cb.store.getDef(x)?.kind !== "type");
      if (!h) throw new Error(`${main}, a command suites run, isn't in the codebase`);
      entries.set(main, writeRunner(h, cb.source(), cb.lay));
    }
    return entries.get(main)!;
  };
  const out = new Map<Suite, TestOutcome>();
  const todo: { s: Suite; of: string }[] = [];
  for (const s of suites) {
    const of = s.hash;
    const r = fresh ? undefined : runs[slot(s)]?.find((kept) => holds(s, kept, of));
    if (r) {
      out.set(s, { path: s.path, result: r.result, cached: true });
      onOutcome?.(out.get(s)!);
    } else todo.push({ s, of });
  }
  // what each suite run here passed with, to keep
  const passed = new Map<Suite, SuiteRun>();
  // side by side, as many as the machine's slots (shared by every tcm running suites on it: `slotPool`) allow; one at
  // a time inside a suite, in the slot its run took
  const pool = !todo.length || inSuiteSlot() ? undefined : slotPool(slotsDir(), machineSlots());
  const what = `tcm test of ${cb.lay.root}${process.env.TCM_BRANCH ? ` on ${process.env.TCM_BRANCH}` : ""}`;
  let next = 0;
  const worker = async () => {
    while (next < todo.length) {
      const { s, of } = todo[next++]!;
      // (the names are as of the run's start, `names`, not of when a slot came free)
      // (a wait said quietly, on stderr: `slotWaits`)
      const release = pool && (await waits.take(pool, what));
      const { result, ran } = await runSuite(cb, s, entry(s), names).finally(() => release?.());
      out.set(s, { path: s.path, result, cached: false });
      onOutcome?.(out.get(s)!);
      const executed = Object.keys(ran).sort();
      if (result.ok && executed.length)
        passed.set(s, {
          of,
          key: keyOf(s),
          format: KEPT_RUN_FORMAT,
          ran: executed,
          code: codeDigest(executed.map((n) => ran[n]!)),
          result,
        });
    }
  };
  await Promise.all(Array.from({ length: Math.min(pool?.size ?? 1, todo.length) }, worker));
  // another run (another branch's, or against another build, side by side) may have kept runs meanwhile: for each
  // suite run here, the runs for the code it ran now are replaced by this one (by nothing, if it failed), and the
  // others kept
  cb.store.transaction(() => {
    const now = unpackRuns(cb.store.getSetting("suites"));
    for (const { s, of } of todo) {
      const k = slot(s);
      const kept = (now[k] ?? []).filter((r) => !holds(s, r, of));
      const run = passed.get(s);
      now[k] = (run ? [run, ...kept] : kept).slice(0, KEPT_RUNS);
    }
    cb.store.setSetting("suites", packRuns(now));
  });
  return suites.map((s) => out.get(s)!);
}

uses

used by