analyzeSources

tcmts.core.analyze.analyzeSources · #4v3sbpmc · term · runs on the server

uses tcmts.core.analyze.importClassifier, which uses node:path

import {
  buildSource, defRefsInOrder, dynamicImportClassifier, externalModule, importClassifier, nodeIndex, paramsOf,
  prefetched, reexportClassifier, signatureOf, substDefs, tableOf, tarjan, typedSites,
} from "#codebase/tcmts/core/analyze";
import type { AnalyzeHost, ProjectDef } from "#codebase/tcmts/core/analyze";
import { canonicalizeSite, declKey } from "#codebase/tcmts/core/canonicalize";
import type { CanonContext, SiteResult } from "#codebase/tcmts/core/canonicalize";
import { docExamples, docTags, exampleNames } from "#codebase/tcmts/core/docs";
import { extract, TcmError } from "#codebase/tcmts/core/extract";
import type { DefSite, ExtractMode } from "#codebase/tcmts/core/extract";
import { hashIR, serializeIR } from "#codebase/tcmts/core/hash";
import type { IR } from "#codebase/tcmts/core/hash";
import type { CompilerOptions } from "#codebase/tcmts/core/program";
import { cacheRefOf, shapeBuilder } from "#codebase/tcmts/core/shape";
import type { TypeShape } from "#codebase/tcmts/core/shape";
import type { TypeChecker } from "#codebase/tcmts/core/ts";
import { depRef } from "#codebase/tcmts/core/types";
import type { Dep, DocRef, Span } from "#codebase/tcmts/core/types";
import {
  isImportClause as ts_isImportClause, isImportDeclaration as ts_isImportDeclaration,
  isImportSpecifier as ts_isImportSpecifier, isNamespaceImport as ts_isNamespaceImport, SyntaxKind as ts_SyntaxKind,
} from "typescript/ast";
import type { Node as ts_Node, SourceFile as ts_SourceFile } from "typescript/ast";
import { SymbolFlags as ts_SymbolFlags } from "typescript/sync";
import type { Symbol as ts_Symbol } from "typescript/sync";

export function analyzeSources(
  sfs: ts_SourceFile[],
  checker: TypeChecker,
  host: AnalyzeHost,
  mode: ExtractMode,
  options: CompilerOptions,
) {
  const errors: string[] = [];
  const warnings: string[] = [];
  const sites: DefSite[] = [];
  const watchSites: DefSite[] = [];
  for (const sf of sfs) {
    try {
      const x = extract(sf, mode, sites.length);
      sites.push(...x.defs);
      watchSites.push(...x.watches);
      warnings.push(...x.warnings);
    } catch (e) {
      if (!(e instanceof TcmError)) throw e;
      errors.push(e.message);
    }
  }
  const declToDef = new Map<string, number>();
  for (const s of sites) for (const d of s.declarations) declToDef.set(declKey(d), s.id);
  const byPath = new Map(sfs.map((sf) => [sf.path as string, sf]));
  // an ambient `declare const x: unique symbol` is a type-only definition, but each one is a distinct symbol
  for (const s of sites)
    if (s.kind !== "type" || s.tag === "Declare")
      s.salt = host.saltFor?.(s.sf.fileName, s.name, s.targetPath);
  const classifiers = new Map(sfs.map((sf) => [sf, importClassifier(sf.fileName, host)]));
  const ctx: CanonContext = {
    checker: prefetched(
      checker,
      [...sites, ...watchSites].flatMap((s) => s.nodes),
    ),
    declToDef,
    projectFiles: new Set(byPath.keys()),
    fileName: (p) => byPath.get(p)?.fileName ?? p,
    nodeOf: (h) => {
      const sf = byPath.get(h.path);
      return sf && nodeIndex(sf).get(h.index);
    },
    allowTransformed: mode === "import",
    classifyImport: (d, meaning) => classifiers.get(d.getSourceFile())!(d, meaning),
    classifyDynamicImport: (spec, at) =>
      dynamicImportClassifier(at.getSourceFile().fileName, host)(spec, at),
    classifyReexport: reexportClassifier(checker, host),
    resolvePath: (path) => {
      for (const meaning of ["type", "value"] as const) {
        try {
          return host.resolveName(path, meaning) || undefined;
        } catch {
          // not that
        }
      }
      return undefined;
    },
    jsxRuntime: (at) => {
      const sf = at.getSourceFile();
      // generated modules name their (aliased) runtime in a pragma; scratch files use the project's setting
      const source =
        /@jsxImportSource\s+(\S+)/.exec(sf.text.slice(0, 2000))?.[1] ??
        (options.jsxImportSource as string | undefined) ??
        "react";
      return externalModule(`${source}/jsx-runtime`, sf.fileName, at, "*", host);
    },
    kindOf: (def) => sites[def]!.kind,
    isLet: (def) => sites[def]!.tag === "Let",
    warn: (message, node) => warnings.push(new TcmError(message, node).message),
  };

  // collect every problem in one pass, so importing a project reports all of them at once
  const results = sites.map((s) => {
    try {
      return canonicalizeSite(s, ctx);
    } catch (e) {
      if (!(e instanceof TcmError)) throw e;
      errors.push(e.message);
      return undefined;
    }
  });
  if (errors.length) return { defs: [], watches: [], warnings, errors };
  const ok = results as SiteResult[];
  // bindings of one destructuring statement must share a component (the statement runs once)
  const groups = new Map<string, DefSite[]>();
  for (const s of sites)
    if (s.group !== undefined)
      groups.set(`${s.sf.fileName} ${s.group}`, [
        ...(groups.get(`${s.sf.fileName} ${s.group}`) ?? []),
        s,
      ]);
  for (const members of groups.values())
    for (const a of members) for (const b of members) if (a !== b) ok[a.id]!.edges.set(b.id, true);
  // a top-level `let` and everything assigning to it are one module (one component)
  for (const s of sites)
    for (const l of ok[s.id]!.writes) if (!ok[l]!.edges.has(s.id)) ok[l]!.edges.set(s.id, false);
  const sccs = tarjan(sites.length, (v) => ok[v]!.edges.keys());

  const hashOf = new Map<number, string>();
  const placed = new Map<number, { componentHash: string; index: number; size: number; ir: IR }>();
  for (const comp of sccs) {
    const inComp = new Set(comp);
    const subst = (ir: IR, cyc: (d: number) => IR) =>
      substDefs(ir, (d) => (inComp.has(d) ? cyc(d) : ["Ref", hashOf.get(d)!]));
    let order = comp;
    if (comp.length > 1) {
      // Unison-style cycle hashing: order members by a hash that ignores intra-cycle references,
      // refine ties by what they reference, and fall back to source order for truly symmetric cycles.
      const pre = new Map(comp.map((id) => [id, hashIR(subst(ok[id]!.ir, () => ["Cyc", "?"]))]));
      const refined = new Map(
        comp.map((id) => [
          id,
          hashIR([
            pre.get(id)!,
            ...defRefsInOrder(ok[id]!.ir)
              .filter((d) => inComp.has(d))
              .map((d) => pre.get(d)!),
          ]),
        ]),
      );
      const cmp = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0);
      order = [...comp].sort(
        (a, b) => cmp(pre.get(a)!, pre.get(b)!) || cmp(refined.get(a)!, refined.get(b)!) || a - b,
      );
    }
    const index = new Map(order.map((id, i) => [id, i]));
    const irs = order.map((id) => subst(ok[id]!.ir, (d) => ["Cyc", String(index.get(d))]));
    const componentHash = hashIR(["Component", ...irs]);
    order.forEach((id, i) => {
      hashOf.set(
        id,
        order.length === 1 ? componentHash : hashIR(["Elem", componentHash, String(i)]),
      );
      placed.set(id, { componentHash, index: i, size: order.length, ir: irs[i]! });
    });
  }

  // the definitions' signatures and parameter schemas, asked all at once (`typedSites`) before their records are made
  let typed: ReturnType<typeof typedSites> | undefined;
  const toRecord = (
    site: DefSite,
    r: SiteResult,
    hash: string,
    p: { componentHash: string; index: number; size: number; ir: IR },
  ): ProjectDef => {
    const src = buildSource(site.pieces, site.sf);
    const spans: Span[] = r.spans.map((s) => {
      const span: Span = { start: src.map(s.start), end: src.map(s.end), pos: s.pos };
      if ("def" in s.target) span.hash = hashOf.get(s.target.def)!;
      else if ("hash" in s.target) span.hash = s.target.hash;
      else span.ext = s.target.ext;
      if (s.shorthand) span.shorthand = s.shorthand;
      if (s.literal) span.literal = true;
      if (s.augmented) span.augmented = true;
      if (s.element) span.element = true;
      return span;
    });
    const deps = new Map<string, Dep>();
    r.spans.forEach((raw, i) => {
      const span = spans[i]!;
      if (span.hash === hash) return;
      const ref = depRef(span);
      const key = `${ref} ${span.pos}`;
      const prev = deps.get(key);
      const called = !!prev?.called || !!raw.called;
      deps.set(key, {
        ref,
        pos: span.pos,
        strict: (prev?.strict ?? false) || raw.strict,
        ...(called ? { called: true } : {}),
      });
    });
    // JSX compiles to calls into its runtime: a dependency on that package, as an import of it would be
    if (r.jsx)
      deps.set(`${r.jsx}/jsx-runtime#*`, {
        ref: `${r.jsx}/jsx-runtime#*`,
        pos: "value",
        strict: false,
      });
    return {
      hash,
      kind: site.kind,
      componentHash: p.componentHash,
      componentIndex: p.index,
      componentSize: p.size,
      ir: serializeIR(p.ir),
      sourceText: src.text,
      spans,
      reservedNames: [...r.reserved].sort(),
      globalNames: [...r.globals].sort(),
      initKind: site.initKind,
      nameHint: site.name,
      nominal: site.nominal,
      targetPath: site.targetPath,
      deps: [...deps.values()],
      signature: typed?.signatures.get(site) ?? signatureOf(site, checker),
      params: typed?.params.has(site) ? typed.params.get(site) : paramsOf(site, checker),
      table: tableOf(
        site,
        checker,
        (d) => {
          const r = classifiers.get(site.sf)?.(d, "value");
          return r?.kind === "hash" ? host.roleOf?.(r.hash) : undefined;
        },
        host.wasTable?.(site.sf.fileName, site.name) ?? false,
      ),
      ...(r.transform ? { transform: true } : {}),
      ...(r.jsx ? { jsx: r.jsx } : {}),
      doc: site.doc,
      file: site.sf.fileName,
      range: {
        start: site.nodes[0]!.getStart(site.sf),
        end: site.nodes.at(-1)!.end,
        ...(site.nameNode
          ? { nameStart: site.nameNode.getStart(site.sf), nameEnd: site.nameNode.end }
          : {}),
      },
    };
  };

  // a type a definition mentions is a codebase type if it's one of these definitions, or a stored one
  const cacheRef = cacheRefOf(checker);
  const shapes = shapeBuilder(checker, (sym) => {
    const d = sym.declarations?.[0];
    const id = d && declToDef.get(declKey(d));
    return id !== undefined ? hashOf.get(id) : cacheRef(sym);
  });
  const shapeOf = (site: DefSite): TypeShape | undefined => {
    if (site.kind !== "term" || site.isWatch || !site.nameNode) return undefined;
    const sym = checker.getSymbolAtLocation(site.nameNode);
    return sym
      ? shapes.typeShape(checker.getTypeOfSymbolAtLocation(sym, site.nameNode))
      : undefined;
  };
  // a doc's links, embeds and examples name definitions: resolve them now, by hash, so the doc follows renames
  // and updates instead of going stale (Unison's docs are code for the same reason)
  const symbolHash = (sym: ts_Symbol): string | undefined => {
    const s = sym.flags & ts_SymbolFlags.Alias ? checker.getAliasedSymbol(sym) : sym;
    const d = s.declarations?.[0];
    const id = d && declToDef.get(declKey(d));
    return id !== undefined ? hashOf.get(id) : cacheRef(s);
  };
  const docRefsOf = (site: DefSite): DocRef[] | undefined => {
    if (!site.doc || site.isWatch) return undefined;
    const tags = docTags(site.doc);
    const examples = docExamples(site.doc);
    if (!tags.length && !examples.length) return undefined;
    const scope = new Map<string, ts_Symbol>();
    const flags =
      ts_SymbolFlags.Value | ts_SymbolFlags.Type | ts_SymbolFlags.Alias | ts_SymbolFlags.Namespace;
    for (const s of checker.getSymbolsInScope(site.nodes[0]!, flags))
      if (!scope.has(s.name)) scope.set(s.name, s);
    const resolve = (name: string, type: boolean): string | undefined => {
      const sym = name.includes(".") ? undefined : scope.get(name);
      const h = sym && symbolHash(sym);
      if (h) return h;
      // `Schema.DateTimeUtc` through `import * as Schema`: that module's definition; `TaggedEnum.WithGenerics`, in the
      // namespace of types beside `TaggedEnum`: the definition holding it
      if (name.includes(".")) {
        const [head, next] = name.split(".") as [string, string];
        const s = scope.get(head);
        const target =
          s && s.flags & ts_SymbolFlags.Alias ? checker.getAliasedSymbol(s) : undefined;
        const mod = target?.declarations?.[0];
        if (mod?.kind === ts_SyntaxKind.SourceFile) {
          const member = checker.getExportsOfModule(target!).find((e) => e.name === next);
          const h = member && symbolHash(member);
          if (h) return h;
        } else if (s && !target) {
          const h = symbolHash(s);
          if (h) return h;
        }
      }
      for (const meaning of type ? (["type", "value"] as const) : (["value", "type"] as const)) {
        try {
          return host.resolveName(name, meaning);
        } catch {
          // not a codebase path either
        }
      }

      // `lib.square` for a definition being added alongside, which has no codebase path yet
      const last = name.slice(name.lastIndexOf(".") + 1);
      const sibling = name.includes(".") ? scope.get(last) : undefined;
      return sibling ? symbolHash(sibling) : undefined;
    };
    const refs: DocRef[] = [];
    for (const t of tags) {
      const h = resolve(t.name, false);
      if (h) refs.push({ start: t.nameStart, end: t.nameEnd, hash: h });
      else
        warnings.push(
          `the doc of ${site.name} refers to '${t.name}', which isn't in scope or in the codebase (${site.sf.fileName})`,
        );
    }
    for (const ex of examples) {
      for (const n of exampleNames(ex.code)) {
        // names an example uses that are definitions, or imports from outside the codebase; globals (Math,
        // console) and the like stay as written
        const sym = scope.get(n.name);
        const h = sym ? symbolHash(sym) : undefined;
        const at = { start: ex.start + n.start, end: ex.start + n.end };
        if (h) refs.push({ ...at, hash: h });
        else {
          const handle =
            sym?.flags && sym.flags & ts_SymbolFlags.Alias ? sym.declarations?.[0] : undefined;
          const decl = handle && ctx.nodeOf(handle);
          if (
            decl &&
            (ts_isImportSpecifier(decl) || ts_isImportClause(decl) || ts_isNamespaceImport(decl))
          ) {
            // (a namespace import of a project file names no definition: `DateTime.make` in an example of Effect's
            // stays as written, as a global does)
            if (!(ts_isNamespaceImport(decl) && importsProjectFile(decl))) {
              const r = ctx.classifyImport(decl, n.type ? "type" : "value");
              if (r.kind === "ext") refs.push({ ...at, ext: r.ext });
            }
          }
        }
      }
    }
    return refs.sort((a, b) => a.start - b.start);
  };
  /** whether an import's module is one of the files analyzed */
  function importsProjectFile(decl: ts_Node): boolean {
    let d: ts_Node | undefined = decl;
    while (d && !ts_isImportDeclaration(d)) d = d.parent;
    const mod = d && checker.getSymbolAtLocation(d.moduleSpecifier)?.declarations?.[0];
    return !!mod && byPath.has(mod.path);
  }
  typed = typedSites(sites, ctx.checker, checker);
  const defs = sites.map((s) => ({
    ...toRecord(s, ok[s.id]!, hashOf.get(s.id)!, placed.get(s.id)!),
    shape: host.shapes === false ? undefined : shapeOf(s),
    docRefs: docRefsOf(s),
  }));

  const watches = watchSites.map((s) => {
    const r = canonicalizeSite(s, ctx);
    const ir = substDefs(r.ir, (d) => ["Ref", hashOf.get(d)!]);
    const componentHash = hashIR(["Component", ir]);
    const rec = toRecord(s, r, componentHash, { componentHash, index: 0, size: 1, ir });
    const expr = s.nodes[0]!;
    return {
      ...rec,
      line: s.sf.getLineAndCharacterOfPosition(expr.getStart(s.sf)).line + 1,
      expression: expr.getText(s.sf),
    };
  });

  return { defs, watches, warnings, errors };
}

uses

used by