extract
tcmts.core.extract.extract · #h3zqygpt · term · runs anywhere
import {
directivePieces, isAnonymousFunction, isInstantiated, jsDocText, MERGED_TAGS, modifierKinds, namespacePragma,
NON_ERASABLE_HINT, pathPragma, setupTargets, SIDE_EFFECT_HINT, strippedStart, TcmError,
} from "#codebase/tcmts/core/extract";
import type { DefSite, Extracted, ExtractMode, Piece } from "#codebase/tcmts/core/extract";
import { isOmitted } from "#codebase/tcmts/core/ts";
import {
getLeadingCommentRanges as ts_getLeadingCommentRanges, isClassDeclaration as ts_isClassDeclaration,
isEnumDeclaration as ts_isEnumDeclaration, isExportAssignment as ts_isExportAssignment,
isExportDeclaration as ts_isExportDeclaration, isExpressionStatement as ts_isExpressionStatement,
isFunctionDeclaration as ts_isFunctionDeclaration, isIdentifier as ts_isIdentifier,
isImportDeclaration as ts_isImportDeclaration, isImportEqualsDeclaration as ts_isImportEqualsDeclaration,
isInterfaceDeclaration as ts_isInterfaceDeclaration, isModuleBlock as ts_isModuleBlock,
isModuleDeclaration as ts_isModuleDeclaration, isNamedExports as ts_isNamedExports,
isStringLiteral as ts_isStringLiteral, isTypeAliasDeclaration as ts_isTypeAliasDeclaration,
isTypeOperatorNode as ts_isTypeOperatorNode, isVariableStatement as ts_isVariableStatement,
NodeFlags as ts_NodeFlags, SyntaxKind as ts_SyntaxKind, tokenToString as ts_tokenToString,
} from "typescript/ast";
import type {
BindingName as ts_BindingName, FunctionDeclaration as ts_FunctionDeclaration,
ImportDeclaration as ts_ImportDeclaration, Node as ts_Node, SourceFile as ts_SourceFile,
} from "typescript/ast";
export function extract(sf: ts_SourceFile, mode: ExtractMode = "scratch", firstId = 0): Extracted {
const defs: DefSite[] = [];
const watches: DefSite[] = [];
const imports: ts_ImportDeclaration[] = [];
const warnings: string[] = [];
/** keyed by `<space> <name>`, space being "type" or "value" */
const byName = new Map<string, DefSite>();
let targetPath: string | undefined;
// a `// tcm: namespace` line below the one a scratch file starts with begins a section: the definitions after it
// go to that namespace
const fileNs =
mode === "scratch"
? /^\/\/\s*tcm:\s*namespace\s+(\S+)/m.exec(sf.text.split("\n").slice(0, 5).join("\n"))?.[1]
: undefined;
let section: string | undefined;
let groups = 0;
/** the current statement's directive comments: they go with its first definition (all of a destructuring's) */
let directives: Piece[] = [];
const push = (fields: Omit<DefSite, "id" | "isWatch" | "sf" | "targetPath">) => {
const site = {
...fields,
targetPath:
targetPath ??
(section !== undefined ? (section ? `${section}.${fields.name}` : fields.name) : undefined),
};
if (directives.length) {
site.pieces = [...directives, ...site.pieces];
if (site.group === undefined) directives = [];
}
// one name may bind a type and a value (`const User = z.object(…)` + `type User = …`); a class is both
const space = site.kind === "type" ? "type" : site.kind === "class" ? "both" : "value";
// declarations that exist only in types merge with a type or class of their name into one definition, as
// TypeScript merges them (`interface Effect<A>` + `declare namespace Effect { type Success<T> … }`,
// `declare const s: unique symbol` + `type s = typeof s`): code imports the name as one thing
const prior = byName.get(`type ${site.name}`);
if (
prior &&
prior.group === undefined &&
prior.targetPath === site.targetPath &&
(site.kind === "type"
? prior.kind === "type" || prior.kind === "class"
: site.kind === "class" && prior.kind === "type" && !byName.has(`value ${site.name}`))
) {
const docs = (ts_getLeadingCommentRanges(sf.text, statement.getFullStart()) ?? []).filter(
(r) => sf.text.startsWith("/**", r.pos),
);
const doc = docs.at(-1);
prior.pieces.push(
{ text: "\n\n" },
...(doc ? [{ start: doc.pos, end: doc.end }, { text: "\n" }] : []),
...site.pieces,
);
prior.nodes.push(...site.nodes);
prior.declarations.push(...site.declarations);
if (MERGED_TAGS.indexOf(site.tag) > MERGED_TAGS.indexOf(prior.tag)) {
prior.tag = site.tag;
prior.nameNode = site.nameNode;
}
prior.doc ??= site.doc;
prior.nominal ||= site.nominal;
if (site.kind === "class") {
prior.kind = "class";
prior.initKind = "strict";
byName.set(`value ${site.name}`, prior);
}
return;
}
for (const s of space === "both" ? ["type", "value"] : [space]) {
if (byName.has(`${s} ${site.name}`)) {
throw new TcmError(
`'${site.name}' is declared more than once. A name binds one definition, or one type plus one value (declaration merging is not supported).`,
site.nodes[0],
);
}
}
const def: DefSite = { ...site, sf, id: firstId + defs.length, isWatch: false };
defs.push(def);
for (const s of space === "both" ? ["type", "value"] : [space])
byName.set(`${s} ${site.name}`, def);
};
/**
* The constant a top-level statement finishes setting up (filling a table, a constructor's `prototype`,
* `Object.freeze`): the one thing it writes into, a constant declared above it. The statement joins its definition,
* so it runs, and is hashed, with it.
*/
const setUp = (st: ts_Node): boolean => {
const targets = setupTargets(st);
const def = targets.size === 1 ? byName.get(`value ${[...targets][0]}`) : undefined;
if (!def || def.tag !== "Const" || def.group !== undefined) return false;
def.nodes.push(st);
def.pieces.push(
{ text: "\n" },
...(ts_getLeadingCommentRanges(sf.text, st.getFullStart()) ?? []).flatMap((c) => [
{ start: c.pos, end: c.end },
{ text: "\n" },
]),
{ start: st.getStart(sf), end: st.end },
);
return true;
};
const statements = sf.statements;
/** the statement being read */
let statement: ts_Node = sf;
for (let i = 0; i < statements.length; i++) {
const st = statements[i]!;
statement = st;
directives = directivePieces(st, sf);
const mods = modifierKinds(st);
// a namespace holding only types (`declare namespace Effect { type Success<T> = … }`) goes on as any namespace;
// `declare global { … }` changes types everywhere, and isn't one
const typesOnlyNamespace =
ts_isModuleDeclaration(st) &&
ts_isIdentifier(st.name) &&
st.name.text !== "global" &&
!!st.body &&
ts_isModuleBlock(st.body) &&
!isInstantiated(st);
if (mods.includes(ts_SyntaxKind.DeclareKeyword) && !typesOnlyNamespace) {
// `declare const brand: unique symbol` (a value that exists only for its type) and `declare class Base` (a
// class to cast a mixin to) are type-only definitions (in a scratch file too, as `tcm edit` writes them);
// anything else ambient (modules, globals) would describe code tcm can't see
// `declare global { … }` adds to what every module sees: a definition of its own, `global`, which nothing
// imports and every check in the codebase includes, as TypeScript includes it in its whole program
if (
ts_isModuleDeclaration(st) &&
ts_isIdentifier(st.name) &&
st.name.text === "global" &&
st.body &&
ts_isModuleBlock(st.body) &&
!isInstantiated(st)
) {
push({
name: "global",
kind: "type",
nodes: [st],
tag: "Augment",
pieces: [{ start: strippedStart(st, sf), end: st.end }],
initKind: "hoisted",
declarations: [],
doc: jsDocText(st, sf),
nameNode: st.name,
nominal: false,
});
continue;
}
// `declare module "./Schema" { namespace Annotations … }` adds to a definition of another module: a definition of
// its own too, `augments`, included in every check as `global` is
if (
ts_isModuleDeclaration(st) &&
ts_isStringLiteral(st.name) &&
st.body &&
ts_isModuleBlock(st.body) &&
!isInstantiated(st)
) {
push({
name: "augments",
kind: "type",
nodes: [st],
tag: "Augment",
pieces: [{ start: strippedStart(st, sf), end: st.end }],
initKind: "hoisted",
declarations: [],
doc: jsDocText(st, sf),
nominal: false,
});
continue;
}
if (ts_isClassDeclaration(st) && st.name) {
push({
name: st.name.text,
kind: "type",
nodes: [st],
tag: "Declare",
pieces: [{ start: strippedStart(st, sf), end: st.end }],
initKind: "hoisted",
declarations: [st],
doc: jsDocText(st, sf),
nameNode: st.name,
nominal: true,
});
continue;
}
if (ts_isVariableStatement(st)) {
const kw = ts_tokenToString(
st.declarationList.flags & ts_NodeFlags.Const
? ts_SyntaxKind.ConstKeyword
: st.declarationList.flags & ts_NodeFlags.Let
? ts_SyntaxKind.LetKeyword
: ts_SyntaxKind.VarKeyword,
);
for (const decl of st.declarationList.declarations) {
if (!ts_isIdentifier(decl.name))
throw new TcmError("ambient destructuring is not supported", decl);
// a `unique symbol` exists only for its type (a brand); anything else declared names what the host provides
// as the code runs (`declare const process: { cwd(): string }`): a value, its module reads it from globalThis
const brand =
!!decl.type &&
ts_isTypeOperatorNode(decl.type) &&
decl.type.operator === ts_SyntaxKind.UniqueKeyword;
push({
name: decl.name.text,
kind: brand ? "type" : "term",
nodes: [decl],
tag: "Declare",
pieces: [{ text: `declare ${kw} ` }, { start: decl.getStart(sf), end: decl.end }],
initKind: brand ? "hoisted" : "strict",
declarations: [decl],
doc: st.declarationList.declarations.length === 1 ? jsDocText(st, sf) : undefined,
nameNode: decl.name,
nominal: true,
});
}
continue;
}
throw new TcmError(
"`declare` is not supported; external code comes in through `import`.",
st,
);
}
const doc = jsDocText(st, sf);
targetPath = pathPragma(st, sf);
const ns = mode === "scratch" ? namespacePragma(st, sf) : undefined;
if (ns !== undefined && (section !== undefined || ns !== fileNs)) section = ns;
if (targetPath && ts_isVariableStatement(st) && st.declarationList.declarations.length > 1) {
throw new TcmError(
"a `// tcm: path` comment names one definition; split this `const` list",
st,
);
}
if (ts_isImportDeclaration(st)) {
imports.push(st);
} else if (ts_isFunctionDeclaration(st)) {
if (!st.name) throw new TcmError("anonymous default export: give the function a name.", st);
// overload signatures followed by the implementation form one definition
const group: ts_FunctionDeclaration[] = [st];
while (!group.at(-1)!.body) {
const next = statements[i + 1];
if (!next || !ts_isFunctionDeclaration(next) || next.name?.text !== st.name.text) break;
group.push(next);
i++;
}
const pieces: Piece[] = [];
for (const [j, fn] of group.entries()) {
if (j > 0) pieces.push({ text: "\n" }, ...directivePieces(fn, sf));
pieces.push({ start: strippedStart(fn, sf), end: fn.end });
}
push({
name: st.name.text,
kind: "term",
nodes: group,
tag: "Function",
pieces,
initKind: "hoisted",
declarations: group,
doc,
nameNode: group.at(-1)!.name,
nominal: true,
});
} else if (ts_isVariableStatement(st)) {
// AwaitUsing is Const|Using, so compare the masked value rather than testing bits
// a top-level `let` is module state (a lazy cache, a config cell): it's nominal, so two identical
// declarations are still two cells, and whatever assigns to it shares its component. (A scratch file takes
// one too: with the codebase the only place code is, what was imported must be editable.)
const blockScope = st.declarationList.flags & ts_NodeFlags.BlockScoped;
if (blockScope === ts_NodeFlags.Let) {
for (const decl of st.declarationList.declarations) {
if (!ts_isIdentifier(decl.name))
throw new TcmError("a top-level `let` with destructuring is not supported", decl);
push({
name: decl.name.text,
kind: "term",
nodes: [decl],
tag: "Let",
pieces: [{ text: "let " }, { start: decl.getStart(sf), end: decl.end }],
// with nothing to evaluate it can go first, before the functions that assign it are declared
initKind: decl.initializer ? "strict" : "hoisted",
declarations: [decl],
doc: st.declarationList.declarations.length === 1 ? doc : undefined,
nameNode: decl.name,
nominal: true,
});
}
continue;
}
if (blockScope !== ts_NodeFlags.Const) {
throw new TcmError(
"top-level `var`/`using` is not supported: use `const` (the value itself may be mutable, e.g. `const cache = new Map()`), or `let` for module state.",
st,
);
}
for (const decl of st.declarationList.declarations) {
if (!ts_isIdentifier(decl.name)) {
// `const { a, b: [c] } = init()`: every bound name is its own definition, but the statement runs
// once, so they share its source and are hashed and emitted together (see DefSite.group)
if (targetPath)
throw new TcmError(
"a `// tcm: path` comment can't name the bindings of a destructuring",
decl,
);
const group = groups++;
const bind = (pattern: ts_BindingName) => {
if (ts_isIdentifier(pattern)) return;
for (const el of pattern.elements) {
if (isOmitted(el)) continue;
if (!ts_isIdentifier(el.name!)) {
bind(el.name!);
continue;
}
push({
name: el.name.text,
kind: "term",
nodes: [decl],
tag: "Destructure",
pieces: [{ text: "const " }, { start: decl.getStart(sf), end: decl.end }],
initKind: "strict",
declarations: [el],
nameNode: el.name,
nominal: !!el.initializer && isAnonymousFunction(el.initializer),
group,
});
}
};
bind(decl.name);
continue;
}
push({
name: decl.name.text,
kind: "term",
nodes: [decl],
tag: "Const",
pieces: [{ text: "const " }, { start: decl.getStart(sf), end: decl.end }],
initKind: "strict",
declarations: [decl],
doc: st.declarationList.declarations.length === 1 ? doc : undefined,
nameNode: decl.name,
nominal: !!decl.initializer && isAnonymousFunction(decl.initializer),
});
}
} else if (ts_isClassDeclaration(st)) {
if (!st.name) throw new TcmError("anonymous default export: give the class a name.", st);
push({
name: st.name.text,
kind: "class",
nodes: [st],
tag: "Class",
pieces: [{ start: strippedStart(st, sf), end: st.end }],
initKind: "strict",
declarations: [st],
doc,
nameNode: st.name,
nominal: true,
});
} else if (ts_isInterfaceDeclaration(st) || ts_isTypeAliasDeclaration(st)) {
push({
name: st.name.text,
kind: "type",
nodes: [st],
tag: ts_isInterfaceDeclaration(st) ? "Interface" : "TypeAlias",
pieces: [{ start: strippedStart(st, sf), end: st.end }],
initKind: "hoisted",
declarations: [st],
doc,
nameNode: st.name,
nominal: false,
});
} else if (ts_isEnumDeclaration(st) && mode === "import") {
// imported code may use syntax Node can't strip: it's compiled when run (see the cache transform hook)
push({
name: st.name.text,
kind: "class", // a type and a value, like a class
nodes: [st],
tag: "Enum",
pieces: [{ start: strippedStart(st, sf), end: st.end }],
initKind: "strict",
declarations: [st],
doc,
nameNode: st.name,
nominal: false,
});
} else if (
ts_isModuleDeclaration(st) &&
(mode === "import" || typesOnlyNamespace) &&
ts_isIdentifier(st.name) &&
st.body &&
ts_isModuleBlock(st.body)
) {
// a namespace is a value (or only a type, if it holds nothing but types: erasable, so a scratch file takes
// one); it can pair with a same-named value, and one holding only types merges with a same-named type
push({
name: st.name.text,
kind: isInstantiated(st) ? "term" : "type",
nodes: [st],
tag: "Namespace",
pieces: [{ start: strippedStart(st, sf), end: st.end }],
initKind: "strict",
declarations: [st],
doc,
nameNode: st.name,
nominal: false,
});
} else if (ts_isEnumDeclaration(st)) {
throw new TcmError(
`enums are not supported in scratch files: ${NON_ERASABLE_HINT} (\`tcm import\` accepts them).`,
st,
);
} else if (ts_isModuleDeclaration(st)) {
throw new TcmError(
`namespaces are not supported${mode === "import" ? " in this form (dotted or ambient)" : " in scratch files"}: ${NON_ERASABLE_HINT}.`,
st,
);
} else if (ts_isImportEqualsDeclaration(st)) {
throw new TcmError("`import x = require(...)` is not supported; use an ES import.", st);
} else if (ts_isExportAssignment(st)) {
// `export default types` in an imported module names a definition it declares, as an export list does: a
// default import of it is a use of that definition
if (mode === "import" && !st.isExportEquals && ts_isIdentifier(st.expression)) continue;
throw new TcmError(
"`export default <expression>` is not supported: give it a name with `const`.",
st,
);
} else if (ts_isExportDeclaration(st)) {
// `export {}` only marks the file as a module
const empty =
!st.moduleSpecifier &&
st.exportClause &&
ts_isNamedExports(st.exportClause) &&
!st.exportClause.elements.length;
if (mode === "import" || empty) continue;
warnings.push(
`ignoring export list at line ${sf.getLineAndCharacterOfPosition(st.getStart(sf)).line + 1}: exports have no meaning in a codebase`,
);
} else if (setUp(st)) {
// (in a scratch file too: a statement assigning into a constant isn't a watch expression)
} else if (ts_isExpressionStatement(st)) {
if (mode === "import")
throw new TcmError(`top-level expression statement: ${SIDE_EFFECT_HINT}.`, st);
const name = `__watch_${watches.length}`;
watches.push({
id: -1 - watches.length,
sf,
name,
kind: "term",
nodes: [st.expression],
tag: "Watch",
pieces: [
{ text: `const ${name} = ` },
{ start: st.expression.getStart(sf), end: st.expression.end },
],
initKind: "strict",
declarations: [],
isWatch: true,
nominal: false,
});
} else if (st.kind === ts_SyntaxKind.EmptyStatement) {
// ignore stray semicolons
} else {
throw new TcmError(
mode === "import"
? `top-level ${ts_SyntaxKind[st.kind]}: ${SIDE_EFFECT_HINT}.`
: `top-level ${ts_SyntaxKind[st.kind]} is not supported: a scratch file contains declarations, imports, and watch expressions.`,
st,
);
}
}
return { defs, watches, imports, warnings };
}
uses
used by