Rebasely Beta: data may be purged

justintypescript-ultra-strict Public

JavaScript · 405 lines · 16.5 KiB · at 83c2f03

// Runs every example against the tsus toolchain and verifies the outcome// declared in each example's example.json.import fs from "node:fs";import path from "node:path";import { fileURLToPath, pathToFileURL } from "node:url";import { spawnSync } from "node:child_process";const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");const cliBin = path.join(repoRoot, "packages/cli/bin/tsus.js");const examplesDir = path.join(repoRoot, "examples");const results = [];// Scratch copies for examples that rewrite files (fix, migrate, init). Kept// inside the repo so compiled output still resolves @tsus/runtime.const tmpRoot = path.join(repoRoot, ".examples-tmp");const freshCopy = (name, from) => {  const dest = path.join(tmpRoot, name);  fs.rmSync(dest, { recursive: true, force: true });  fs.mkdirSync(dest, { recursive: true });  if (from) {    fs.cpSync(from, dest, {      recursive: true,      filter: (src) => !/(^|[\\/])(dist|node_modules)([\\/]|$)/.test(path.relative(from, src)),    });  }  return dest;};const buildAndRun = (dir, expected) => {  const rb = runNode([cliBin, "build", dir], repoRoot);  if (rb.status !== 0) {    return `build failed:\n${(rb.stderr || rb.stdout || "").trim()}`;  }  const rr = runNode([path.join(dir, "dist/main.js")], dir);  if (rr.status !== 0) {    return `run failed:\n${(rr.stderr || "").trim()}`;  }  return checkLines(rr.stdout, expected);};const sameFile = (actualPath, goldenPath) => {  const actual = fs.readFileSync(actualPath, "utf8");  const golden = fs.readFileSync(goldenPath, "utf8");  return actual === golden ? "" : `${actualPath} differs from ${goldenPath}`;};// Each expectation returns a failure string ("" = pass).const expectFix = (name, dir, manifest) => {  const work = freshCopy(name, dir);  const rf = runNode([cliBin, "fix", work, "--json"], repoRoot);  const parsed = JSON.parse(rf.stdout || "{}");  if (!parsed.ok) {    return `fix left diagnostics: ${(parsed.diagnostics ?? []).map((d) => d.code).join(", ")}`;  }  const applied = (parsed.applied ?? []).map((a) => a.code).sort();  const want = [...(manifest.applied ?? [])].sort();  if (JSON.stringify(applied) !== JSON.stringify(want)) {    return `applied ${JSON.stringify(applied)}, expected ${JSON.stringify(want)}`;  }  for (const [file, golden] of Object.entries(manifest.golden ?? {})) {    const diff = sameFile(path.join(work, file), path.join(dir, golden));    if (diff) {      return diff;    }  }  return buildAndRun(work, manifest.output);};const expectMigrate = (name, dir, manifest) => {  const work = freshCopy(name, dir);  for (const item of manifest.migrate) {    const rm = runNode([cliBin, "migrate", path.join(work, item.input), "--json"], repoRoot);    if (rm.status === 2) {      return `migrate ${item.input} failed:\n${(rm.stderr || "").trim()}`;    }    const report = JSON.parse(rm.stdout);    const todo = report.unconverted.map((u) => (u.kind === "check" ? `check:${u.code}` : u.kind));    if (JSON.stringify(todo) !== JSON.stringify(item.todo)) {      return `${item.input}: todo ${JSON.stringify(todo)}, expected ${JSON.stringify(item.todo)}`;    }    const out = path.join(work, item.input.replace(/\.ts$/, ".tsus"));    const diff = sameFile(out, path.join(dir, item.golden));    if (diff) {      return diff;    }  }  for (const file of manifest.remove ?? []) {    fs.rmSync(path.join(work, file));  }  // Only src/ is built; partial migrations elsewhere are compared, not run.  return buildAndRun(work, manifest.output);};const expectInit = (name, _dir, manifest) => {  const work = freshCopy(name);  const ri = runNode([cliBin, "init", work], repoRoot);  if (ri.status !== 0) {    return `init failed:\n${(ri.stderr || ri.stdout || "").trim()}`;  }  for (const file of manifest.files) {    if (!fs.existsSync(path.join(work, file))) {      return `init did not create ${file}`;    }  }  const primer = sameFile(path.join(work, "AGENTS.md"), path.join(repoRoot, "AGENTS.md"));  if (primer) {    return primer;  }  const again = runNode([cliBin, "init", work, "--json"], repoRoot);  if (JSON.parse(again.stdout).written.length !== 0) {    return "a second init overwrote existing files";  }  return buildAndRun(work, manifest.output);};// --- v2: plain .ts checked by `tsus check --v2`, built by esbuild, audited by `tsus audit` ---const esbuildBin = path.join(repoRoot, "node_modules/.bin/esbuild");const bundle = (name, entry) => {  const out = path.join(tmpRoot, name, "bundle.mjs");  fs.mkdirSync(path.dirname(out), { recursive: true });  const r = spawnSync(esbuildBin, [entry, "--bundle", "--platform=node", "--format=esm", "--target=node22", `--outfile=${out}`, "--log-level=warning"], { encoding: "utf8" });  return r.status === 0 ? { out } : { error: `esbuild failed:\n${(r.stderr || r.stdout || "").trim()}` };};const checkV2Json = (dir) => {  const r = runNode([cliBin, "check", "--v2", dir, "--json"], repoRoot);  return { status: r.status, diagnostics: JSON.parse(r.stdout || "{}").diagnostics ?? [], stderr: r.stderr };};const tsFilesUnder = (dir) =>  fs.existsSync(dir) ? fs.readdirSync(dir, { recursive: true }).filter((f) => /\.ts$/.test(f)).map((f) => path.join(dir, f)) : [];// Lines ending in `expect CODE ...` (or `finding category ...`): what the checker (or the audit)// must report on that line, and the only lines where it may report anything.const markersIn = (dir, word) => {  const want = new Map();  for (const file of tsFilesUnder(dir)) {    fs.readFileSync(file, "utf8").split("\n").forEach((line, i) => {      const m = new RegExp(`(?://|--)\\s*${word}\\s+([\\w\\- ]+)$`).exec(line);      if (m) want.set(`${path.basename(file)}:${i + 1}`, m[1].trim().split(/\s+/));    });  }  return want;};const compareMarkers = (want, got) => {  for (const [at, items] of want) {    const seen = [...(got.get(at) ?? [])];    for (const item of items) {      const k = seen.indexOf(item);      if (k === -1) return `${at}: expected ${item}, got ${(got.get(at) ?? []).join(", ") || "nothing"}`;      seen.splice(k, 1);    }  }  for (const [at, items] of got) {    const allowed = want.get(at);    const extra = items.filter((i) => !allowed?.includes(i));    if (extra.length > 0) return `${at}: unexpected ${extra.join(", ")}`;  }  return "";};const byLine = (items, keyOf) => {  const got = new Map();  for (const d of items) {    const at = `${path.basename(d.file)}:${d.line}`;    got.set(at, [...(got.get(at) ?? []), keyOf(d)]);  }  return got;};const expectV2Run = (name, dir, manifest) => {  const c = checkV2Json(dir);  if (c.status !== 0) return `check --v2 failed: ${c.diagnostics.map((d) => `${d.code}@${d.file}:${d.line}`).join(", ") || c.stderr}`;  const entry = path.join(dir, manifest.entry);  const b = bundle(name, entry);  if (b.error) return b.error;  for (const [how, args] of [["esbuild bundle", [b.out]], ["node type stripping", [entry]]]) {    const r = runNode(args, dir);    if (r.status !== 0) return `${how}: run failed:\n${(r.stderr || "").trim()}`;    const diff = checkLines(r.stdout, manifest.output);    if (diff) return `${how}: ${diff}`;  }  return "";};const expectV2CheckFail = (_name, dir, manifest) => {  const c = checkV2Json(dir);  if (c.status === 0) return "check --v2 unexpectedly passed";  const seen = new Set(c.diagnostics.map((d) => d.code));  const missing = (manifest.codes ?? []).filter((code) => !seen.has(code));  if (missing.length > 0) return `missing expected codes ${missing.join(", ")}; saw ${[...seen].join(", ")}`;  return manifest.markers ? compareMarkers(markersIn(path.join(dir, "src"), "expect"), byLine(c.diagnostics, (d) => d.code)) : "";};// The proof: the same module in plain strict TS and in v2, six planted bugs each.const expectV2Demo = (name, dir, manifest) => {  const bugLines = (sub) => {    const lines = new Map();    for (const file of tsFilesUnder(path.join(dir, sub, "src"))) {      fs.readFileSync(file, "utf8").split("\n").forEach((line, i) => {        const m = /\/\/ BUG ([\d, ]+)/.exec(line);        if (m) lines.set(`${path.basename(file)}:${i + 1}`, m[1].split(/[, ]+/).filter(Boolean));      });    }    return lines;  };  const bugs = Object.keys(manifest.bugs);  // 1. Plain strict TypeScript (strict, noUncheckedIndexedAccess) reports nothing.  const plainFiles = tsFilesUnder(path.join(dir, manifest.plain, "src"));  const ts = spawnSync(process.execPath, [path.join(repoRoot, "node_modules/typescript/bin/tsc"), "--noEmit", "--strict", "--noUncheckedIndexedAccess", "--target", "es2022", "--module", "esnext", "--moduleResolution", "bundler", "--lib", "es2023,dom", "--types", "node", ...plainFiles], { encoding: "utf8" });  if (ts.status !== 0) return `plain TypeScript is supposed to pass: ${ts.stdout.trim()}`;  const plainBugs = new Set([...bugLines(manifest.plain).values()].flat());  const missingPlain = bugs.filter((b) => !plainBugs.has(b));  if (missingPlain.length) return `plain/ has no line marked BUG ${missingPlain.join(", ")}`;  // 2. v2 reports every planted bug with the expected code, and nothing on unmarked lines.  const c = checkV2Json(path.join(dir, manifest.v2));  const lines = bugLines(manifest.v2);  const caught = new Set();  for (const d of c.diagnostics) {    const at = `${path.basename(d.file)}:${d.line}`;    const marked = lines.get(at);    if (!marked) return `v2 reported ${d.code} at ${at}, which is not a planted bug`;    for (const b of marked) if ((manifest.bugs[b] ?? []).includes(d.code)) caught.add(b);  }  const missed = bugs.filter((b) => !caught.has(b));  if (missed.length) return `v2 missed BUG ${missed.join(", ")}`;  // 3. The fixed version passes, bundles with esbuild, and the harness shows both at runtime.  const f = checkV2Json(path.join(dir, manifest.fixed));  if (f.status !== 0) return `fixed/ does not pass check --v2: ${f.diagnostics.map((d) => `${d.code}@${d.file}:${d.line}`).join(", ")}`;  const b = bundle(name, path.join(dir, manifest.harness));  if (b.error) return b.error;  const r = runNode([b.out], dir);  if (r.status !== 0) return `harness failed:\n${(r.stderr || "").trim()}`;  return checkLines(r.stdout, manifest.output);};const fileHashes = (dir) => new Map(tsFilesUnder(dir).map((f) => [f, fs.readFileSync(f, "utf8")]));const expectAudit = (_name, dir, manifest) => {  const before = fileHashes(dir);  const files = manifest.files.map((f) => path.join(dir, f));  const r = runNode([cliBin, "audit", ...files, "--scope", path.join(dir, manifest.scope), "--json"], repoRoot);  if (r.status !== 0) return `audit failed:\n${(r.stderr || "").trim()}`;  const after = fileHashes(dir);  for (const [f, text] of before) if (after.get(f) !== text) return `audit changed ${f}`;  const report = JSON.parse(r.stdout);  return compareMarkers(markersIn(path.join(dir, "src"), "finding"), byLine(report.findings, (x) => x.category));};const SCRATCH_EXPECTATIONS = {  fix: expectFix, migrate: expectMigrate, init: expectInit,  "v2-run": expectV2Run, "v2-check-fail": expectV2CheckFail, "v2-demo": expectV2Demo, audit: expectAudit,};const runNode = (args, cwd) =>  spawnSync(process.execPath, args, { cwd, encoding: "utf8", timeout: 30_000 });const checkLines = (stdout, expected) => {  const lines = stdout.trim().split("\n").map((l) => l.trim());  if (lines.length !== expected.length) {    return `expected ${expected.length} lines, got ${lines.length}: ${JSON.stringify(lines)}`;  }  for (let i = 0; i < expected.length; i += 1) {    if (lines[i] !== expected[i]) {      return `line ${i + 1}: expected ${JSON.stringify(expected[i])}, got ${JSON.stringify(lines[i])}`;    }  }  return "";};for (const name of fs.readdirSync(examplesDir).sort()) {  const dir = path.join(examplesDir, name);  const manifestPath = path.join(dir, "example.json");  if (!fs.existsSync(manifestPath)) {    continue;  }  const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));  let failure = "";  if (manifest.expect === "command") {    // Framework examples: install their own deps, run their own build/run    // command, then assert on stdout and/or produced files.    if (manifest.install && !fs.existsSync(path.join(dir, "node_modules"))) {      const ri = spawnSync("npm", ["install", "--no-audit", "--no-fund"], {        cwd: dir,        encoding: "utf8",        timeout: 600_000,      });      if (ri.status !== 0) {        failure = `npm install failed:\n${(ri.stderr || ri.stdout || "").trim().slice(-2000)}`;      }    }    if (failure === "") {      const rc = spawnSync(manifest.command[0], manifest.command.slice(1), {        cwd: dir,        encoding: "utf8",        timeout: 600_000,      });      if (rc.status !== 0) {        failure = `command failed:\n${(rc.stderr || rc.stdout || "").trim().slice(-2000)}`;      } else {        if (manifest.output) {          failure = checkLines(rc.stdout, manifest.output);        }        for (const fc of manifest.fileContains ?? []) {          const produced = path.join(dir, fc.file);          if (!fs.existsSync(produced)) {            failure = `expected output file ${fc.file} was not produced`;          } else if (!fs.readFileSync(produced, "utf8").includes(fc.contains)) {            failure = `${fc.file} does not contain ${JSON.stringify(fc.contains)}`;          }        }      }    }    results.push({ name, ok: failure === "", failure });    continue;  }  if (SCRATCH_EXPECTATIONS[manifest.expect]) {    failure = SCRATCH_EXPECTATIONS[manifest.expect](name, dir, manifest);    results.push({ name, ok: failure === "", failure });    continue;  }  if (manifest.expect === "check-fail") {    const r = runNode([cliBin, "check", dir, "--json"], repoRoot);    if (r.status === 0) {      failure = "check unexpectedly passed";    } else {      const parsed = JSON.parse(r.stdout || "{}");      const seen = new Set((parsed.diagnostics ?? []).map((d) => d.code));      const missing = (manifest.codes ?? []).filter((c) => !seen.has(c));      if (missing.length > 0) {        failure = `missing expected codes ${missing.join(", ")}; saw ${[...seen].join(", ") || "none"}`;      }      // Optional exact expectations: each listed diagnostic must be reported,      // matching every field it names (code, file, line, col, tsCode). This      // pins positions mapped back to the .tsus source.      for (const want of manifest.diagnostics ?? []) {        const hit = (parsed.diagnostics ?? []).some((d) =>          Object.entries(want).every(([key, value]) => d[key] === value),        );        if (!hit && failure === "") {          const seenAt = (parsed.diagnostics ?? []).map((d) => `${d.code}@${d.line}:${d.col}`).join(", ");          failure = `expected diagnostic ${JSON.stringify(want)}; saw ${seenAt || "none"}`;        }      }      // Codes listed under "fixable" must come with a machine-applicable fix.      for (const code of manifest.fixable ?? []) {        const withFix = (parsed.diagnostics ?? []).filter((d) => d.code === code && d.fix && d.fix.edits.length > 0);        if (withFix.length === 0) {          failure = `${code} has no fix`;        }      }    }  } else {    const rb = runNode([cliBin, "build", dir], repoRoot);    if (rb.status !== 0) {      failure = `build failed:\n${(rb.stderr || rb.stdout || "").trim()}`;    } else if (manifest.expect === "run") {      const rr = runNode([path.join(dir, "dist/main.js")], dir);      if (rr.status !== 0) {        failure = `run failed:\n${(rr.stderr || "").trim()}`;      } else {        failure = checkLines(rr.stdout, manifest.output);      }    } else if (manifest.expect === "worker") {      const entry = pathToFileURL(path.join(dir, "dist", manifest.entry)).href;      try {        const mod = await import(entry);        for (const req of manifest.requests) {          const res = await mod.default.fetch(new Request(`http://worker.local${req.path}`));          const body = await res.text();          if (res.status !== req.status || body !== req.body) {            failure = `${req.path}: expected ${req.status} ${JSON.stringify(req.body)}, got ${res.status} ${JSON.stringify(body)}`;            break;          }        }      } catch (error) {        failure = `worker import failed: ${error.message}`;      }    } else {      failure = `unknown expectation ${manifest.expect}`;    }  }  results.push({ name, ok: failure === "", failure });}let passed = 0;for (const r of results) {  if (r.ok) {    passed += 1;    console.log(`PASS ${r.name}`);  } else {    console.log(`FAIL ${r.name}\n     ${r.failure.replaceAll("\n", "\n     ")}`);  }}console.log(`\n${passed}/${results.length} examples passed`);process.exit(passed === results.length ? 0 : 1);