startFakeAtproto

tcmts.cli.test.e2e.examples.startFakeAtproto · #bfvfku7z · term · runs on the server

uses node:crypto

The AT Protocol as the userinput example uses it, on one local server, so its end-to-end test needs no network: Constellation's backlinks (blue.microcosm.links.*, /links/all), Slingshot's records and identities, the appview's profiles and typeahead, the image service, a PLC directory, and a PDS for each account (password sessions, records, blobs) with its OAuth authorization server (PAR, authorize, token; DPoP-bound tokens, nonces asked for, PKCE checked; authorizing signs in as the login hint at once, as if they'd said yes), its records signed (com.atproto.sync.getRecord's CAR, with each account's #atproto key in its DID document: a P-256 Multikey, or a legacy uncompressed secp256k1 key, legacyKey), a relay's listReposByCollection, the appview's follows (app.bsky.graph.getFollows, from app.bsky.graph.follow records), and a Jetstream (v2's subscribeEvents, a websocket) of every record written. Backlinks are every { uri } a record holds, at its path (subject.uri). Allows any origin.

import type { Keep } from "#codebase/tcmts/cli/test/e2e/contributions";
import { legacyMultibase, signedRecordCar, wsFrame } from "#codebase/tcmts/cli/test/e2e/examples";
import type { FakeAccount, FakeAtproto, FakeRecord } from "#codebase/tcmts/cli/test/e2e/examples";
import { P256PrivateKeyExportable, Secp256k1PrivateKeyExportable } from "@atcute/crypto";
import { createHash, createPublicKey, verify } from "node:crypto";
import http from "node:http";
import type { Duplex } from "node:stream";

export async function startFakeAtproto(keep: Keep, accounts: FakeAccount[]): Promise<FakeAtproto> {
  const records = new Map<string, FakeRecord>();
  /** each account's repository signing key, and (for the DID document) its verification method */
  const keys = new Map<
    string,
    { key: { sign(data: Uint8Array): Promise<Uint8Array> }; method: Record<string, string> }
  >();
  for (const a of accounts) {
    if (a.legacyKey) {
      const key = await Secp256k1PrivateKeyExportable.createKeypair();
      keys.set(a.did, {
        key,
        method: {
          id: "#atproto",
          type: "EcdsaSecp256k1VerificationKey2019",
          controller: a.did,
          publicKeyMultibase: legacyMultibase(await key.exportPublicKey("raw")),
        },
      });
    } else {
      const key = await P256PrivateKeyExportable.createKeypair();
      keys.set(a.did, {
        key,
        method: {
          id: `${a.did}#atproto`,
          type: "Multikey",
          controller: a.did,
          publicKeyMultibase: await key.exportPublicKey("multikey"),
        },
      });
    }
  }
  /** records whose getRecord is signed by some other key, as a forged one would be */
  const forged = new Set<string>();
  /** Jetstream: every event, and who's listening (with what they asked for) */
  const stream: Record<string, unknown>[] = [];
  const listening = new Set<{ socket: Duplex; wants: (p: Record<string, unknown>) => boolean }>();
  let seq = 0;
  const emit = (
    did: string,
    operation: string,
    collection: string,
    rkey: string,
    record?: FakeRecord,
  ) => {
    const time = new Date().toISOString();
    const payload: Record<string, unknown> = {
      $type: "network.bsky.jetstream.subscribeEvents#commit",
      seq: ++seq,
      did,
      time,
      witnessedAt: time,
      rev: tid(),
      operation,
      collection,
      rkey,
      ...(record ? { record: record.value, cid: record.cid } : {}),
    };
    stream.push(payload);
    for (const l of listening)
      if (l.wants(payload)) l.socket.write(wsFrame(JSON.stringify({ $type: "message", payload })));
  };
  const blobs = new Map<string, { mime: string; data: Buffer }>();
  let tick = 0;
  const tid = () => `3m${(Date.now() * 1000 + tick++).toString(32).padStart(11, "0")}`;
  const cidOf = (data: string | Buffer) =>
    `bafyrei${createHash("sha256").update(data).digest("hex").slice(0, 52)}`;
  const put = (
    did: string,
    collection: string,
    rkey: string,
    value: Record<string, unknown>,
  ): FakeRecord => {
    const v = { $type: collection, ...value };
    const r = { uri: `at://${did}/${collection}/${rkey}`, cid: cidOf(JSON.stringify(v)), value: v };
    const was = records.has(r.uri);
    records.set(r.uri, r);
    emit(did, was ? "update" : "create", collection, rkey, r);
    return r;
  };
  /** every `{ uri: "at://…" }` in a value, by its path */
  const links = (v: unknown, path: string, out: [string, string][] = []): [string, string][] => {
    if (Array.isArray(v)) v.forEach((x) => links(x, path, out));
    else if (v && typeof v === "object") {
      const o = v as Record<string, unknown>;
      if (typeof o.uri === "string" && o.uri.startsWith("at://")) out.push([`${path}uri`, o.uri]);
      for (const [k, x] of Object.entries(o)) if (k !== "uri") links(x, `${path}${k}.`, out);
    }
    return out;
  };
  const remove = (uri: string) => {
    if (!records.delete(uri)) return;
    const p = parts(uri);
    emit(p.did, "delete", p.collection, p.rkey);
  };
  const parts = (uri: string) => {
    const [, , did, collection, rkey] = uri.split("/");
    return { did: did!, collection: collection!, rkey: rkey! };
  };
  const backlinksTo = (subject: string) =>
    [...records.values()].flatMap((r) =>
      links(r.value, "")
        .filter(([, u]) => u === subject)
        .map(([p]) => ({ r, source: `${parts(r.uri).collection}:${p}` })),
    );
  const sha256 = (s: string) => createHash("sha256").update(s).digest("base64url");
  const NONCE = "fake-nonce";
  /** OAuth: pushed requests by request_uri, codes, and tokens (each bound to a DPoP key's thumbprint) */
  const oauth = {
    requests: new Map<string, Record<string, string> & { jkt: string }>(),
    codes: new Map<string, Record<string, string> & { jkt: string; did: string }>(),
    access: new Map<string, { did: string; jkt: string }>(),
    refresh: new Map<string, { did: string; jkt: string; client_id: string; scope: string }>(),
    signIns: 0,
    refreshes: 0,
  };
  let issued = 0;
  /** A request's DPoP proof, checked: its key's thumbprint, "nonce" if it hasn't the nonce, or undefined if it's bad. */
  const dpop = (
    req: http.IncomingMessage,
    pathname: string,
    accessToken?: string,
  ): string | undefined => {
    const [h, p, sig] = String(req.headers.dpop ?? "").split(".");
    if (!h || !p || !sig) return undefined;
    try {
      const header = JSON.parse(Buffer.from(h, "base64url").toString()) as {
        typ: string;
        alg: string;
        jwk: Record<string, string>;
      };
      const claims = JSON.parse(Buffer.from(p, "base64url").toString()) as Record<string, unknown>;
      const key = createPublicKey({ key: header.jwk, format: "jwk" });
      if (header.typ !== "dpop+jwt" || header.alg !== "ES256") return undefined;
      if (
        !verify(
          "sha256",
          Buffer.from(`${h}.${p}`),
          { key, dsaEncoding: "ieee-p1363" },
          Buffer.from(sig, "base64url"),
        )
      )
        return undefined;
      if (claims.htm !== req.method || claims.htu !== `http://127.0.0.1:${port}${pathname}`)
        return undefined;
      if (accessToken && claims.ath !== sha256(accessToken)) return undefined;
      if (claims.nonce !== NONCE) return "nonce";
      const { crv, kty, x, y } = header.jwk;
      return sha256(JSON.stringify({ crv, kty, x, y }));
    } catch {
      return undefined;
    }
  };
  const account = (req: http.IncomingMessage, pathname: string) => {
    const auth = String(req.headers.authorization ?? "");
    const bearer = /^Bearer acc-(.+)$/.exec(auth)?.[1];
    if (bearer) return accounts.find((a) => a.did === bearer);
    const token = /^DPoP (.+)$/.exec(auth)?.[1];
    const t = token ? oauth.access.get(token) : undefined;
    if (!t) return undefined;
    const jkt = dpop(req, pathname, token);
    if (jkt === "nonce") return "nonce" as const;
    return jkt === t.jkt ? accounts.find((a) => a.did === t.did) : undefined;
  };
  const server = http.createServer(async (req, res) => {
    res.setHeader("access-control-allow-origin", "*");
    // (what the browser asks for, by name: `*` doesn't cover Authorization)
    res.setHeader(
      "access-control-allow-headers",
      String(req.headers["access-control-request-headers"] ?? "*"),
    );
    res.setHeader("access-control-allow-methods", "GET, POST, OPTIONS");
    res.setHeader("access-control-expose-headers", "*");
    if (req.method === "OPTIONS") return res.end();
    const url = new URL(req.url ?? "/", "http://x");
    const q = url.searchParams;
    const chunks: Buffer[] = [];
    for await (const c of req) chunks.push(c as Buffer);
    const raw = Buffer.concat(chunks);
    const body = () => JSON.parse(raw.toString("utf8") || "{}") as Record<string, any>;
    const json = (status: number, v: unknown) => {
      res.writeHead(status, { "content-type": "application/json" });
      res.end(JSON.stringify(v));
    };
    const method = url.pathname.replace(/^\/xrpc\//, "");
    const here = `http://127.0.0.1:${port}`;
    const form = () => Object.fromEntries(new URLSearchParams(raw.toString("utf8")));
    const askNonce = (status: number) => {
      res.setHeader("dpop-nonce", NONCE);
      if (status === 401) res.setHeader("www-authenticate", 'DPoP error="use_dpop_nonce"');
      return json(status, { error: "use_dpop_nonce", error_description: "use the server's nonce" });
    };
    const tokens = (did: string, jkt: string, client_id: string, scope: string) => {
      const access = `at-${++issued}`;
      const refresh = `rt-${issued}`;
      oauth.access.set(access, { did, jkt });
      oauth.refresh.set(refresh, { did, jkt, client_id, scope });
      return json(200, {
        access_token: access,
        token_type: "DPoP",
        refresh_token: refresh,
        expires_in: 3600,
        scope,
        sub: did,
      });
    };
    switch (url.pathname) {
      case "/.well-known/oauth-protected-resource":
        return json(200, { resource: here, authorization_servers: [here] });
      case "/.well-known/oauth-authorization-server":
        return json(200, {
          issuer: here,
          authorization_endpoint: `${here}/oauth/authorize`,
          token_endpoint: `${here}/oauth/token`,
          pushed_authorization_request_endpoint: `${here}/oauth/par`,
          require_pushed_authorization_requests: true,
          dpop_signing_alg_values_supported: ["ES256"],
          code_challenge_methods_supported: ["S256"],
          client_id_metadata_document_supported: true,
        });
      case "/oauth/par": {
        const f = form();
        const jkt = dpop(req, url.pathname);
        if (jkt === "nonce") return askNonce(400);
        if (!jkt) return json(400, { error: "invalid_dpop_proof" });
        const loopback =
          (f.client_id ?? "").startsWith("http://localhost?") &&
          new URL(f.client_id!).searchParams.get("redirect_uri") === f.redirect_uri;
        if (
          !loopback ||
          !/^http:\/\/127\.0\.0\.1:\d+\//.test(f.redirect_uri ?? "") ||
          f.code_challenge_method !== "S256"
        )
          return json(400, {
            error: "invalid_request",
            error_description: "a loopback client, coming back to 127.0.0.1, with PKCE",
          });
        const request_uri = `urn:ietf:params:oauth:request_uri:req-${++issued}`;
        oauth.requests.set(request_uri, { ...f, jkt });
        return json(201, { request_uri, expires_in: 300 });
      }
      case "/oauth/authorize": {
        const r = oauth.requests.get(q.get("request_uri") ?? "");
        if (!r || r.client_id !== q.get("client_id"))
          return json(400, { error: "invalid_request" });
        oauth.requests.delete(q.get("request_uri")!);
        const a = accounts.find((x) => x.handle === r.login_hint || x.did === r.login_hint);
        const back = new URL(r.redirect_uri!);
        // (the answer in the query, or in the fragment if asked: a browser client's)
        const answer = new URLSearchParams({ state: r.state ?? "", iss: here });
        if (!a) answer.set("error", "access_denied");
        else {
          const code = `code-${++issued}`;
          oauth.codes.set(code, { ...r, did: a.did });
          answer.set("code", code);
          oauth.signIns++;
        }
        if (r.response_mode === "fragment") back.hash = answer.toString();
        else for (const [k, v] of answer) back.searchParams.set(k, v);
        res.writeHead(302, { location: back.href });
        return res.end();
      }
      case "/oauth/token": {
        const f = form();
        const jkt = dpop(req, url.pathname);
        if (jkt === "nonce") return askNonce(400);
        if (!jkt) return json(400, { error: "invalid_dpop_proof" });
        if (f.grant_type === "authorization_code") {
          const c = oauth.codes.get(f.code ?? "");
          oauth.codes.delete(f.code ?? "");
          if (
            !c ||
            c.jkt !== jkt ||
            c.client_id !== f.client_id ||
            c.redirect_uri !== f.redirect_uri ||
            sha256(f.code_verifier ?? "") !== c.code_challenge
          )
            return json(400, { error: "invalid_grant" });
          return tokens(c.did, jkt, c.client_id!, c.scope!);
        }
        if (f.grant_type === "refresh_token") {
          const t = oauth.refresh.get(f.refresh_token ?? "");
          oauth.refresh.delete(f.refresh_token ?? "");
          if (!t || t.jkt !== jkt || t.client_id !== f.client_id)
            return json(400, { error: "invalid_grant" });
          oauth.refreshes++;
          return tokens(t.did, jkt, t.client_id, t.scope);
        }
        return json(400, { error: "unsupported_grant_type" });
      }
    }
    // (a DID's document, its colons as written or escaped: `/did%3Aplc%3A…`)
    if (decodeURIComponent(url.pathname).startsWith("/did:")) {
      const a = accounts.find((x) => x.did === decodeURIComponent(url.pathname.slice(1)));
      return a
        ? json(200, {
            "@context": [
              "https://www.w3.org/ns/did/v1",
              a.legacyKey
                ? "https://w3id.org/security/suites/secp256k1-2019/v1"
                : "https://w3id.org/security/multikey/v1",
            ],
            id: a.did,
            alsoKnownAs: [`at://${a.handle}`],
            service: [
              { id: "#atproto_pds", type: "AtprotoPersonalDataServer", serviceEndpoint: here },
            ],
            verificationMethod: [keys.get(a.did)!.method],
          })
        : json(404, { message: "DID not registered" });
    }
    const who = account(req, url.pathname);
    if (who === "nonce") return askNonce(401);
    const me = who;
    const writable = (repo: string) => me && me.did === repo;
    switch (method) {
      case "com.atproto.identity.resolveHandle": {
        const a = accounts.find((x) => x.handle === q.get("handle"));
        return a ? json(200, { did: a.did }) : json(400, { error: "HandleNotFound" });
      }
      case "blue.microcosm.identity.resolveMiniDoc": {
        const id = q.get("identifier");
        const a = accounts.find((x) => x.did === id || x.handle === id);
        return a
          ? json(200, { did: a.did, handle: a.handle, pds: `http://127.0.0.1:${port}` })
          : json(400, { error: "NotFound" });
      }
      case "com.atproto.repo.getRecord": {
        const r = records.get(`at://${q.get("repo")}/${q.get("collection")}/${q.get("rkey")}`);
        return r ? json(200, r) : json(400, { error: "RecordNotFound" });
      }
      case "com.atproto.sync.listReposByCollection": {
        const dids = [
          ...new Set(
            [...records.values()]
              .filter((r) => parts(r.uri).collection === q.get("collection"))
              .map((r) => parts(r.uri).did),
          ),
        ];
        return json(200, { repos: dids.map((did) => ({ did })) });
      }
      case "com.atproto.sync.getRecord": {
        const uri = `at://${q.get("did")}/${q.get("collection")}/${q.get("rkey")}`;
        const r = records.get(uri);
        const signer = keys.get(q.get("did") ?? "");
        if (!r || !signer) return json(400, { error: "RecordNotFound" });
        const key = forged.has(uri) ? await P256PrivateKeyExportable.createKeypair() : signer.key;
        const car = await signedRecordCar(
          q.get("did")!,
          `${q.get("collection")}/${q.get("rkey")}`,
          r.value,
          key,
          tid(),
        );
        res.writeHead(200, { "content-type": "application/vnd.ipld.car" });
        return res.end(car);
      }
      case "com.atproto.repo.listRecords": {
        const prefix = `at://${q.get("repo")}/${q.get("collection")}/`;
        return json(200, {
          records: [...records.values()].filter((r) => r.uri.startsWith(prefix)).reverse(),
        });
      }
      case "blue.microcosm.links.getBacklinks":
      case "blue.microcosm.links.getBacklinksCount": {
        const dids = q.getAll("did");
        const found = backlinksTo(q.get("subject") ?? "").filter(
          (l) =>
            l.source === q.get("source") && (!dids.length || dids.includes(parts(l.r.uri).did)),
        );
        if (method.endsWith("Count")) return json(200, { total: found.length });
        return json(200, {
          total: found.length,
          records: found.map((l) => parts(l.r.uri)),
          cursor: null,
        });
      }
      case "/links/all": {
        const out: Record<string, Record<string, { records: number; distinct_dids: number }>> = {};
        for (const l of backlinksTo(q.get("target") ?? "")) {
          const [collection, p] = l.source.split(":") as [string, string];
          const slot = ((out[collection] ??= {})[`.${p}`] ??= { records: 0, distinct_dids: 0 });
          slot.records++;
          slot.distinct_dids++;
        }
        return json(200, { links: out });
      }
      case "app.bsky.actor.getProfiles":
        return json(200, {
          profiles: accounts
            .filter((a) => q.getAll("actors").includes(a.did))
            .map((a) => ({ did: a.did, handle: a.handle, displayName: a.displayName })),
        });
      case "app.bsky.graph.getFollows": {
        const actor = accounts.find((a) => a.did === q.get("actor") || a.handle === q.get("actor"));
        const follows = [...records.values()]
          .filter((r) => r.uri.startsWith(`at://${actor?.did}/app.bsky.graph.follow/`))
          .map((r) => accounts.find((a) => a.did === (r.value as { subject?: string }).subject))
          .filter((a) => a !== undefined)
          .map((a) => ({ did: a.did, handle: a.handle, displayName: a.displayName }));
        return actor
          ? json(200, { subject: { did: actor.did, handle: actor.handle }, follows })
          : json(400, { error: "InvalidRequest" });
      }
      case "app.bsky.actor.searchActorsTypeahead":
        return json(200, {
          actors: accounts
            .filter((a) => a.handle.startsWith(q.get("q") ?? ""))
            .map((a) => ({ did: a.did, handle: a.handle, displayName: a.displayName })),
        });
      case "com.atproto.server.createSession": {
        const b = body();
        const a = accounts.find(
          (x) => (x.did === b.identifier || x.handle === b.identifier) && x.password === b.password,
        );
        return a
          ? json(200, {
              did: a.did,
              handle: a.handle,
              accessJwt: `acc-${a.did}`,
              refreshJwt: `ref-${a.did}`,
              active: true,
            })
          : json(401, {
              error: "AuthenticationRequired",
              message: "Invalid identifier or password",
            });
      }
      case "com.atproto.server.refreshSession":
      case "com.atproto.server.deleteSession":
        return json(200, {});
      case "com.atproto.repo.uploadBlob": {
        if (!me) return json(401, { error: "AuthenticationRequired" });
        const cid = cidOf(raw);
        blobs.set(cid, {
          mime: String(req.headers["content-type"] ?? "application/octet-stream"),
          data: raw,
        });
        return json(200, {
          blob: {
            $type: "blob",
            ref: { $link: cid },
            mimeType: String(req.headers["content-type"]),
            size: raw.length,
          },
        });
      }
      case "com.atproto.repo.createRecord":
      case "com.atproto.repo.putRecord": {
        const b = body();
        if (!writable(b.repo)) return json(401, { error: "AuthenticationRequired" });
        const r = put(b.repo, b.collection, b.rkey ?? tid(), b.record);
        return json(200, { uri: r.uri, cid: r.cid });
      }
      case "com.atproto.repo.deleteRecord": {
        const b = body();
        if (!writable(b.repo)) return json(401, { error: "AuthenticationRequired" });
        remove(`at://${b.repo}/${b.collection}/${b.rkey}`);
        return json(200, {});
      }
      case "com.atproto.repo.applyWrites": {
        const b = body();
        if (!writable(b.repo)) return json(401, { error: "AuthenticationRequired" });
        for (const w of b.writes as {
          $type: string;
          collection: string;
          rkey?: string;
          value?: Record<string, unknown>;
        }[]) {
          if (w.$type.endsWith("#delete")) remove(`at://${b.repo}/${w.collection}/${w.rkey}`);
          else put(b.repo, w.collection, w.rkey ?? tid(), w.value ?? {});
        }
        return json(200, {});
      }
    }
    const blob = /^\/blob\/([^/]+)\/([^/]+)\//.exec(url.pathname);
    if (blob) {
      const b = blobs.get(decodeURIComponent(blob[2]!));
      if (!b) return json(404, { error: "NotFound" });
      res.writeHead(200, { "content-type": b.mime });
      return res.end(b.data);
    }
    json(404, { error: "MethodNotImplemented", message: url.pathname });
  });
  // Jetstream v2: subscribeEvents, its xrpc.v1.json frames one JSON object each; `collections` (with `prefix.*`) and
  // `kinds` filter commits; `cursor` replays from a seq, or (from 1e15) a time in microseconds
  server.on("upgrade", (req: http.IncomingMessage, socket: Duplex) => {
    const url = new URL(req.url ?? "/", "http://x");
    if (url.pathname !== "/xrpc/network.bsky.jetstream.subscribeEvents")
      return void socket.destroy();
    const accept = createHash("sha1")
      .update(`${req.headers["sec-websocket-key"]}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`)
      .digest("base64");
    const asked = String(req.headers["sec-websocket-protocol"] ?? "")
      .split(",")
      .map((x) => x.trim());
    socket.write(
      `HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ${accept}\r\n${asked.includes("xrpc.v1.json") ? "Sec-WebSocket-Protocol: xrpc.v1.json\r\n" : ""}\r\n`,
    );
    const collections = url.searchParams.getAll("collections");
    const kinds = url.searchParams.getAll("kinds");
    const cursor = Number(url.searchParams.get("cursor") ?? Number.NaN);
    const wants = (p: Record<string, unknown>) =>
      (!kinds.length || kinds.includes("commit")) &&
      (!collections.length ||
        collections.some((c) =>
          c.endsWith(".*") ? String(p.collection).startsWith(c.slice(0, -1)) : p.collection === c,
        ));
    const since = (p: Record<string, unknown>) =>
      cursor >= 1e15 ? Date.parse(String(p.time)) * 1000 >= cursor : (p.seq as number) >= cursor;
    if (!Number.isNaN(cursor))
      for (const p of stream)
        if (wants(p) && since(p))
          socket.write(wsFrame(JSON.stringify({ $type: "message", payload: p })));
    const l = { socket, wants };
    listening.add(l);
    // (what the client sends: a close is answered and ends it, a ping is answered; nothing else is read)
    socket.on("data", (b: Buffer) => {
      const op = b[0]! & 0x0f;
      if (op === 0x8) socket.end(Buffer.from([0x88, 0]));
      else if (op === 0x9) {
        const len = b[1]! & 0x7f;
        const mask = b.subarray(2, 6);
        const data = Buffer.from(b.subarray(6, 6 + len).map((x, i) => x ^ mask[i % 4]!));
        socket.write(Buffer.concat([Buffer.from([0x8a, data.length]), data]));
      }
    });
    socket.on("close", () => listening.delete(l));
    socket.on("error", () => listening.delete(l));
  });
  await new Promise<void>((r) => server.listen(0, "127.0.0.1", r));
  const port = (server.address() as { port: number }).port;
  keep({
    kill: () => {
      for (const l of listening) l.socket.destroy();
      server.close();
    },
  } as never);
  return {
    url: `http://127.0.0.1:${port}`,
    records,
    seed: put,
    oauth,
    forge: (uri) => void forged.add(uri),
    unsigned: (did, collection, rkey, value) =>
      emit(did, "create", collection, rkey, {
        uri: `at://${did}/${collection}/${rkey}`,
        cid: cidOf(JSON.stringify(value)),
        value: { $type: collection, ...value },
      }),
  };
}

uses

used by