migrateFile

tcmts.cli.migrate.migrateFile · #nltgddsk · term · runs on the server

uses node:sqlite

Move a codebase's database (a file nothing else has open) to this tcm's hash format, in place, in one transaction. A pure function of what's in it, so every copy of a codebase (a clone, a server) migrates to the same hashes and they go on syncing:

import { remapHashes } from "#codebase/tcmts/cli/migrate";
import type { Migrated } from "#codebase/tcmts/cli/migrate";
import { hashIR, IR_VERSION, rehashComponent } from "#codebase/tcmts/core/hash";
import type { IR } from "#codebase/tcmts/core/hash";
import { upgradeIR } from "#codebase/tcmts/core/migrate";
import { causalHash, namespaceHash } from "#codebase/tcmts/store";
import type { NamespaceBody } from "#codebase/tcmts/store";
import { DatabaseSync } from "node:sqlite";

export function migrateFile(file: string): Migrated {
  const to = IR_VERSION;
  const db = new DatabaseSync(file);
  db.exec("PRAGMA foreign_keys = OFF; BEGIN IMMEDIATE");
  try {
    const from = (
      db.prepare("SELECT DISTINCT ir_version AS v FROM components ORDER BY v").all() as {
        v: number;
      }[]
    ).map((r) => r.v);
    type Def = { hash: string; kind: string; comp: string; i: number; ir: string; v: number };
    const defs = db
      .prepare(
        "SELECT d.hash, d.kind, d.component_hash AS comp, d.component_index AS i, d.ir, c.ir_version AS v FROM definitions d JOIN components c ON c.hash = d.component_hash",
      )
      .all() as Def[];
    const byComp = new Map<string, Def[]>();
    for (const d of defs) byComp.set(d.comp, [...(byComp.get(d.comp) ?? []), d]);
    for (const ms of byComp.values()) ms.sort((a, b) => a.i - b.i);
    const compOf = new Map(defs.map((d) => [d.hash, d.comp]));
    const refs = (ir: unknown, out: string[] = []): string[] => {
      if (Array.isArray(ir)) {
        if (ir[0] === "Ref" && typeof ir[1] === "string") out.push(ir[1]);
        else for (const c of ir) refs(c, out);
      }
      return out;
    };
    // what each component uses before it (components make no cycles)
    const order: string[] = [];
    const placed = new Set<string>();
    for (const start of byComp.keys()) {
      const stack: [string, boolean][] = [[start, false]];
      while (stack.length) {
        const [c, done] = stack.pop()!;
        if (done) {
          if (!placed.has(c)) {
            placed.add(c);
            order.push(c);
          }
          continue;
        }
        if (placed.has(c)) continue;
        stack.push([c, true]);
        for (const m of byComp.get(c)!)
          for (const r of refs(JSON.parse(m.ir))) {
            const dep = compOf.get(r);
            if (dep && dep !== c && !placed.has(dep)) stack.push([dep, false]);
          }
      }
    }
    const defMap = new Map<string, string>();
    const compMap = new Map<string, string>();
    const next = new Map<string, { ir: IR; index: number }>();
    let rewritten = 0;
    const swap = (h: string) => defMap.get(h) ?? h;
    const subst = (ir: IR): IR =>
      Array.isArray(ir)
        ? ir[0] === "Ref" && typeof ir[1] === "string"
          ? ["Ref", swap(ir[1])]
          : ir.map(subst)
        : ir;
    for (const c of order) {
      const ms = byComp.get(c)!;
      const irs = ms.map((m) => {
        const was = JSON.parse(m.ir) as IR;
        const now = upgradeIR(was, m.v, to);
        if (JSON.stringify(now) !== m.ir) rewritten++;
        return now;
      });
      // a file kept as written hashes its content (and the hashes it links to), as itself
      if (ms[0]!.kind === "file") {
        const ir = subst(irs[0]!);
        const h = hashIR(ir);
        defMap.set(ms[0]!.hash, h);
        compMap.set(c, h);
        next.set(ms[0]!.hash, { ir, index: 0 });
        continue;
      }
      // (a cycle whose members its hashes can't order is ordered as written, which is how it's stored)
      const r =
        rehashComponent(irs, swap) ??
        (() => {
          const swapped = irs.map(subst);
          const componentHash = hashIR(["Component", ...swapped]);
          return {
            irs: swapped,
            componentHash,
            hashes: swapped.map((_, i) => hashIR(["Elem", componentHash, String(i)])),
            index: swapped.map((_, i) => i),
          };
        })();
      compMap.set(c, r.componentHash);
      ms.forEach((m, i) => {
        const at = r.index[i]!;
        defMap.set(m.hash, r.hashes[at]!);
        next.set(m.hash, { ir: r.irs[at]!, index: at });
      });
    }

    // the definitions, under their new hashes (no new hash is an old one: the format is in every hash)
    const sizes = new Map(
      (
        db.prepare("SELECT hash, size, created_at FROM components").all() as {
          hash: string;
          size: number;
          created_at: number;
        }[]
      ).map((r) => [r.hash, r]),
    );
    const putComp = db.prepare(
      "INSERT OR IGNORE INTO components (hash, size, ir_version, created_at) VALUES (?, ?, ?, ?)",
    );
    for (const [old, now] of compMap)
      putComp.run(now, sizes.get(old)!.size, to, sizes.get(old)!.created_at);
    const rows = db.prepare("SELECT * FROM definitions").all() as Record<string, string | number>[];
    const putDef = db.prepare(
      "INSERT OR IGNORE INTO definitions (hash, kind, component_hash, component_index, ir, source_text, ref_spans, reserved_names, init_kind, name_hint, nominal) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
    );
    for (const r of rows) {
      const h = String(r.hash);
      const n = next.get(h)!;
      putDef.run(
        defMap.get(h)!,
        r.kind!,
        compMap.get(String(r.component_hash))!,
        n.index,
        JSON.stringify(n.ir),
        r.source_text!,
        JSON.stringify(remapHashes(JSON.parse(String(r.ref_spans)), defMap)),
        r.reserved_names!,
        r.init_kind!,
        r.name_hint!,
        r.nominal!,
      );
    }
    const putDep = db.prepare(
      "INSERT OR IGNORE INTO deps (from_hash, to_ref, ref_kind, strict) VALUES (?, ?, ?, ?)",
    );
    for (const d of db.prepare("SELECT * FROM deps").all() as Record<string, string | number>[])
      if (defMap.has(String(d.from_hash)))
        putDep.run(
          defMap.get(String(d.from_hash))!,
          remapHashes(d.to_ref, defMap) as string,
          d.ref_kind!,
          d.strict!,
        );
    const putSig = db.prepare(
      "INSERT OR IGNORE INTO signatures (hash, display, ts_version) VALUES (?, ?, ?)",
    );
    for (const s of db.prepare("SELECT * FROM signatures").all() as Record<string, string>[])
      if (defMap.has(s.hash!)) putSig.run(defMap.get(s.hash!)!, s.display!, s.ts_version!);
    const putMeta = db.prepare(
      "INSERT OR IGNORE INTO metadata (hash, key, value) VALUES (?, ?, ?)",
    );
    for (const m of db.prepare("SELECT * FROM metadata").all() as Record<string, string>[]) {
      // (what tcm worked out about code by its own code's hash: caches, made again)
      if (!defMap.has(m.hash!) || /^(effects|reaches) /.test(m.key!)) continue;
      let value = m.value!;
      try {
        value = JSON.stringify(remapHashes(JSON.parse(value), defMap));
      } catch {
        value = remapHashes(value, defMap) as string;
      }
      putMeta.run(defMap.get(m.hash!)!, m.key!, value);
    }
    for (const [table, col] of [
      ["definitions", "hash"],
      ["deps", "from_hash"],
      ["signatures", "hash"],
      ["metadata", "hash"],
      ["components", "hash"],
    ] as const) {
      const del = db.prepare(`DELETE FROM ${table} WHERE ${col} = ?`);
      // (one already in this format, and using only such, keeps its hash: it stays)
      for (const [old, now] of table === "components" ? compMap : defMap)
        if (old !== now) del.run(old);
    }

    // history, from the bottom up
    const nsMap = new Map<string, string>();
    const getNs = db.prepare("SELECT body FROM namespaces WHERE hash = ?");
    const putNs = db.prepare("INSERT OR IGNORE INTO namespaces (hash, body) VALUES (?, ?)");
    const rewriteNs = (h: string): string => {
      const known = nsMap.get(h);
      if (known) return known;
      const row = getNs.get(h) as { body: string } | undefined;
      if (!row) return h;
      const [, names, children, texts] = JSON.parse(row.body) as NamespaceBody;
      // (the text versions names show are kept by their own hash, which follows from their text: only the
      // definitions they're of move)
      const shown = [
        ...new Map(
          (texts ?? []).map(([n, d, t]): [string, [string, string, string]] => [
            `${n} ${swap(d)}`,
            [n, swap(d), t],
          ]),
        ).values(),
      ].sort(([a, x], [b, y]) => (a < b ? -1 : a > b ? 1 : x < y ? -1 : x > y ? 1 : 0));
      const node: NamespaceBody = [
        "ns",
        names.map(([n, hs]): [string, string[]] => [n, [...new Set(hs.map(swap))].sort()]),
        children.map(([n, c]): [string, string] => [n, rewriteNs(c)]),
      ];
      const body = JSON.stringify(shown.length ? [...node, shown] : node);
      const now = namespaceHash(body);
      putNs.run(now, body);
      nsMap.set(h, now);
      return now;
    };
    const causals = db.prepare("SELECT hash, namespace, ts, command FROM causals").all() as {
      hash: string;
      namespace: string;
      ts: number;
      command: string;
    }[];
    const parentsOf = new Map<string, string[]>();
    for (const p of db.prepare("SELECT child, parent FROM causal_parents").all() as {
      child: string;
      parent: string;
    }[])
      parentsOf.set(p.child, [...(parentsOf.get(p.child) ?? []), p.parent]);
    const byHash = new Map(causals.map((c) => [c.hash, c]));
    const causalMap = new Map<string, string>();
    const putCausal = db.prepare(
      "INSERT OR IGNORE INTO causals (hash, namespace, ts, command) VALUES (?, ?, ?, ?)",
    );
    const putParent = db.prepare(
      "INSERT OR IGNORE INTO causal_parents (child, parent) VALUES (?, ?)",
    );
    for (const start of causals) {
      const stack: [string, boolean][] = [[start.hash, false]];
      while (stack.length) {
        const [h, done] = stack.pop()!;
        if (causalMap.has(h)) continue;
        const c = byHash.get(h);
        if (!c) {
          causalMap.set(h, h);
          continue;
        }
        const ps = parentsOf.get(h) ?? [];
        if (!done) {
          stack.push([h, true]);
          for (const p of ps) if (!causalMap.has(p)) stack.push([p, false]);
          continue;
        }
        const ns = rewriteNs(c.namespace);
        const parents = ps.map((p) => causalMap.get(p) ?? p);
        const now = causalHash(ns, parents);
        putCausal.run(now, ns, c.ts, c.command);
        for (const p of new Set(parents)) putParent.run(now, p);
        causalMap.set(h, now);
      }
    }
    const keepNs = new Set(nsMap.values());
    const delNs = db.prepare("DELETE FROM namespaces WHERE hash = ?");
    for (const old of nsMap.keys()) if (!keepNs.has(old)) delNs.run(old);
    const keepCausal = new Set(causalMap.values());
    for (const old of causalMap.keys())
      if (!keepCausal.has(old)) {
        db.prepare("DELETE FROM causals WHERE hash = ?").run(old);
        db.prepare("DELETE FROM causal_parents WHERE child = ?").run(old);
      }

    // what's keyed by hashes
    const all = new Map([...defMap, ...causalMap]);
    const each = (sql: string, f: (r: Record<string, string | number | null>) => void) => {
      for (const r of db.prepare(sql).all() as Record<string, string | number | null>[]) f(r);
    };
    const mapped = (v: string | number | null | undefined) =>
      typeof v === "string" ? (all.get(v) ?? v) : (v ?? null);
    each("SELECT id, head FROM branches", (r) =>
      db.prepare("UPDATE branches SET head = ? WHERE id = ?").run(mapped(r.head), r.id!),
    );
    each("SELECT package, version, causal FROM releases", (r) =>
      db
        .prepare("UPDATE releases SET causal = ? WHERE package = ? AND version = ?")
        .run(mapped(r.causal), r.package!, r.version!),
    );
    each("SELECT id, merged_from, merged_as FROM contributions", (r) =>
      db
        .prepare("UPDATE contributions SET merged_from = ?, merged_as = ? WHERE id = ?")
        .run(mapped(r.merged_from), mapped(r.merged_as), r.id!),
    );
    each("SELECT id, about FROM contribution_comments", (r) =>
      db
        .prepare("UPDATE contribution_comments SET about = ? WHERE id = ?")
        .run(mapped(r.about), r.id!),
    );
    each("SELECT branch_id, path, hash FROM names", (r) =>
      db
        .prepare("UPDATE names SET hash = ? WHERE branch_id = ? AND path = ? AND hash = ?")
        .run(mapped(r.hash), r.branch_id!, r.path!, r.hash!),
    );
    each("SELECT id, old_hash, new_hash FROM name_log", (r) =>
      db
        .prepare("UPDATE name_log SET old_hash = ?, new_hash = ? WHERE id = ?")
        .run(mapped(r.old_hash), mapped(r.new_hash), r.id!),
    );
    each("SELECT branch_id, old_hash, new_hash FROM patches", (r) => {
      if (defMap.has(String(r.old_hash)) && defMap.has(String(r.new_hash)))
        db.prepare(
          "UPDATE patches SET old_hash = ?, new_hash = ? WHERE branch_id = ? AND old_hash = ?",
        ).run(mapped(r.old_hash), mapped(r.new_hash), r.branch_id!, r.old_hash!);
      else
        db.prepare("DELETE FROM patches WHERE branch_id = ? AND old_hash = ?").run(
          r.branch_id!,
          r.old_hash!,
        );
    });
    each("SELECT branch_id, hash FROM todo", (r) =>
      db
        .prepare("UPDATE todo SET hash = ? WHERE branch_id = ? AND hash = ?")
        .run(mapped(r.hash), r.branch_id!, r.hash!),
    );
    // results that go with definitions this has; a watch expression's (no definition of its own) is made again
    for (const table of ["test_results", "lint_results"])
      each(`SELECT DISTINCT hash FROM ${table}`, (r) => {
        if (defMap.has(String(r.hash)))
          db.prepare(`UPDATE ${table} SET hash = ? WHERE hash = ?`).run(mapped(r.hash), r.hash!);
        else db.prepare(`DELETE FROM ${table} WHERE hash = ?`).run(r.hash!);
      });
    db.exec(
      "DELETE FROM watch_results; DELETE FROM settings WHERE key IN ('verified', 'suites') OR key LIKE 'views_at%' OR key LIKE 'shim_stamp%'",
    );
    db.exec("COMMIT");
    return {
      from,
      to,
      definitions: defMap.size,
      rewritten,
      causals: causalMap.size,
      namespaces: nsMap.size,
    };
  } catch (e) {
    db.exec("ROLLBACK");
    throw e;
  } finally {
    db.close();
  }
}

uses

used by