Rebasely Beta: data may be purged

justintypescript-ultra-strict Public

JavaScript · 204 lines · 10.2 KiB · at 358de06

#!/usr/bin/env node// Writes evals/RESULTS.md from one or more run directories.////   node evals/report.mjs                       # every run under evals/results//   node evals/report.mjs evals/results/<run>   # just these runs//   node evals/report.mjs --stdout <run>        # print instead of writingimport fs from "node:fs";import path from "node:path";import { evalsDir, repoRoot } from "./lib/compile.mjs";const args = process.argv.slice(2);const toStdout = args.includes("--stdout");const resultsRoot = path.join(evalsDir, "results");let runDirs = args.filter((a) => !a.startsWith("--")).map((a) => path.resolve(a));if (runDirs.length === 0) {  runDirs = fs.existsSync(resultsRoot)    ? fs        .readdirSync(resultsRoot)        .map((n) => path.join(resultsRoot, n))        .filter((d) => fs.existsSync(path.join(d, "meta.json")))    : [];}const loadRun = (dir) => {  const meta = JSON.parse(fs.readFileSync(path.join(dir, "meta.json"), "utf8"));  const file = path.join(dir, "results.jsonl");  const records = fs.existsSync(file)    ? fs        .readFileSync(file, "utf8")        .split("\n")        .filter(Boolean)        .map((l) => JSON.parse(l))    : [];  records.sort((a, b) => a.task.localeCompare(b.task) || a.lang.localeCompare(b.lang));  return { dir, meta, records };};// Runs where every job failed at the provider (no model access) carry no// information; leave them out.const runs = runDirs  .map(loadRun)  .filter((r) => r.records.some((rec) => rec.status !== "provider_error" && rec.status !== "aborted"))  .sort((a, b) => b.meta.startedAt.localeCompare(a.meta.startedAt));const pct = (n, d) => (d === 0 ? "n/a" : `${Math.round((100 * n) / d)}%`);const frac = (n, d) => `${n}/${d} (${pct(n, d)})`;const avg = (xs) => (xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length);const money = (x) => `$${x.toFixed(x < 1 ? 4 : 2)}`;const isModelRun = (meta) => meta.provider !== "reference" && meta.provider !== "selftest";const summarize = (records) => {  const scored = records.filter((r) => r.status !== "provider_error" && r.status !== "aborted");  const withTests = scored.filter((r) => r.tests);  return {    jobs: records.length,    scored: scored.length,    providerErrors: records.length - scored.length,    pass1: scored.filter((r) => r.passFirstTry).length,    passFinal: scored.filter((r) => r.passFinal).length,    compiledFirst: scored.filter((r) => r.compiledFirstTry).length,    neverCompiled: scored.filter((r) => r.status === "compile_failed").length,    tasksCaughtAtCompile: scored.filter((r) => r.compileRejections > 0).length,    compileRejections: scored.reduce((n, r) => n + r.compileRejections, 0),    compileDiagnostics: scored.reduce((n, r) => n + r.compileDiagnostics, 0),    rescuedByRepair: scored.filter((r) => r.compileRejections > 0 && r.passFinal).length,    avgRepairs: avg(scored.map((r) => r.repairRounds)),    tasksFailingAtRuntime: withTests.filter((r) => !r.tests.allPassed).length,    failingTestCases: withTests.reduce((n, r) => n + r.tests.fail, 0),    totalTestCases: withTests.reduce((n, r) => n + r.tests.total, 0),    cost: records.reduce((n, r) => n + r.costUsd, 0),    outputTokens: records.reduce((n, r) => n + r.usage.outputTokens, 0),    avgSeconds: avg(scored.map((r) => r.ms / 1000)),  };};const codeCounts = (records) => {  const counts = new Map();  for (const r of records) {    for (const a of r.attempts) {      for (const c of a.codes) {        counts.set(c, (counts.get(c) ?? 0) + 1);      }    }  }  return [...counts.entries()].sort((a, b) => b[1] - a[1]);};const cell = (r) => {  if (!r) {    return "not run";  }  if (r.status === "provider_error" || r.status === "aborted") {    return `${r.status}`;  }  const tests = r.tests ? `, tests ${r.tests.pass}/${r.tests.total}` : "";  const word = r.passFirstTry ? "pass@1" : r.passFinal ? "pass after repair" : r.status === "compile_failed" ? "never compiled" : "fails tests";  return `${word} (${r.attempts.length} attempt${r.attempts.length === 1 ? "" : "s"}${tests})`;};const renderRun = ({ dir, meta, records }) => {  const langs = meta.langs;  const byLang = Object.fromEntries(langs.map((l) => [l, summarize(records.filter((r) => r.lang === l))]));  const lines = [];  const date = meta.startedAt.slice(0, 10);  lines.push(`## ${meta.label[0].toUpperCase()}${meta.label.slice(1)}: ${date}, ${meta.provider === "claude-cli" || meta.provider === "anthropic" ? meta.model : meta.provider}`);  lines.push("");  if (!isModelRun(meta)) {    lines.push(      `**No model was called in this run.** The \`${meta.provider}\` provider feeds the hand-written reference solutions${meta.provider === "selftest" ? " (with a planted compile error on the first attempt, to exercise the repair loop)" : ""} through the full pipeline. It shows the harness and the hidden tests work end to end. It says nothing about the thesis.`,    );    lines.push("");  }  lines.push(    `Run \`${meta.runId}\` (${path.relative(repoRoot, dir)}): provider \`${meta.provider}\`, model \`${meta.model}\`, effort ${meta.effort}, up to ${meta.maxRepairs} repair rounds, ${meta.tasks.length} task(s), repo commit \`${meta.repoCommit}\`${meta.tsusCliDirty ? " (compiler had uncommitted changes)" : ""}, Node ${meta.node}. Total cost ${money(meta.totalCostUsd ?? 0)}.${meta.aborted ? ` Aborted: ${meta.aborted}.` : ""}`,  );  lines.push("");  lines.push(`| Metric | ${langs.map((l) => `.${l}`).join(" | ")} |`);  lines.push(`|---|${langs.map(() => "---").join("|")}|`);  const row = (label, f) => lines.push(`| ${label} | ${langs.map((l) => f(byLang[l])).join(" | ")} |`);  row("Tasks scored", (s) => `${s.scored}${s.providerErrors ? ` (+${s.providerErrors} provider errors)` : ""}`);  row("pass@1 (first attempt compiles and passes every hidden test)", (s) => frac(s.pass1, s.scored));  row("Pass after compiler-guided repair", (s) => frac(s.passFinal, s.scored));  row("Compiled on first attempt", (s) => frac(s.compiledFirst, s.scored));  row("Tasks where the compiler caught something", (s) => `${s.tasksCaughtAtCompile}`);  row("Compile-time catches (rejected attempts / diagnostics)", (s) => `${s.compileRejections} / ${s.compileDiagnostics}`);  row("Tasks rescued by repair (rejected, then passed)", (s) => `${s.rescuedByRepair}`);  row("Never compiled within the repair budget", (s) => `${s.neverCompiled}`);  row("Average repair rounds", (s) => s.avgRepairs.toFixed(2));  row("Tasks failing at runtime (hidden tests)", (s) => `${s.tasksFailingAtRuntime}`);  row("Failing hidden test cases", (s) => `${s.failingTestCases} of ${s.totalTestCases}`);  row("Cost", (s) => money(s.cost));  row("Output tokens", (s) => `${s.outputTokens}`);  row("Average wall time per task (model calls only)", (s) => `${s.avgSeconds.toFixed(1)} s`);  lines.push("");  for (const l of langs) {    const codes = codeCounts(records.filter((r) => r.lang === l));    if (codes.length > 0) {      lines.push(`Diagnostic codes seen in .${l} attempts: ${codes.map(([c, n]) => `${c} x${n}`).join(", ")}.`);      lines.push("");    }  }  lines.push(`<details><summary>Per-task detail</summary>`);  lines.push("");  lines.push(`| Task | ${langs.map((l) => `.${l}`).join(" | ")} |`);  lines.push(`|---|${langs.map(() => "---").join("|")}|`);  const taskIds = [...new Set(records.map((r) => r.task))].sort();  for (const id of taskIds) {    lines.push(`| ${id} | ${langs.map((l) => cell(records.find((r) => r.task === id && r.lang === l))).join(" | ")} |`);  }  lines.push("");  const failing = records.filter((r) => r.tests && !r.tests.allPassed);  if (failing.length > 0) {    lines.push("Failing hidden tests:");    lines.push("");    for (const r of failing) {      lines.push(`- ${r.task} (.${r.lang}): ${r.tests.failures.join("; ") || "see attempts/*.tests.txt"}`);    }    lines.push("");  }  const errors = records.filter((r) => r.error);  if (errors.length > 0) {    lines.push("Provider errors:");    lines.push("");    for (const r of errors) {      lines.push(`- ${r.task} (.${r.lang}): ${r.error}`);    }    lines.push("");  }  lines.push(`</details>`);  lines.push("");  return lines.join("\n");};const taskCount = fs.readdirSync(path.join(evalsDir, "tasks")).filter((n) => fs.existsSync(path.join(evalsDir, "tasks", n, "task.json"))).length;const modelRuns = runs.filter((r) => isModelRun(r.meta));const header = `# tsus eval resultsGenerated by \`node evals/report.mjs\` from the runs in \`evals/results/\`. Method and caveats are in [evals/README.md](README.md).The question: when a model writes the same small program in plain TypeScript (\`tsc --strict\`) and in TypeScript-Ultra-Strict, which version is correct more often? Each of the ${taskCount} tasks has a prompt, a signature per language, and hidden \`node:test\` tests the model never sees. The compiler's errors are fed back for up to N repair rounds; test results never are.How to read the metrics:- **pass@1**: the first attempt compiled and passed every hidden test.- **Pass after repair**: some attempt within the repair budget compiled, and that version passed every hidden test.- **Compile-time catches**: attempts the compiler rejected. In .tsus these include the ban rules (TSUS001 to TSUS005) and macro errors. A catch is only good news if the repaired version then passes; "tasks rescued by repair" counts those.- **Failing at runtime**: the solution compiled but at least one hidden test failed. These are the bugs that would have shipped.${  modelRuns.length === 0    ? "**No model run has been recorded yet.** The sections below come from the harness self-test, which uses the reference solutions instead of a model. See evals/README.md for how to run the pilot.\n"    : `**Sample size warning.** The largest model run below covers ${Math.max(...modelRuns.map((r) => r.meta.tasks.length))} task(s), one sample each, with one model. Differences of a few tasks between languages are within noise. Treat these numbers as a pilot, not as evidence.\n`}`;const body = runs.length === 0 ? "No runs found.\n" : runs.map(renderRun).join("\n");const out = `${header}\n${body}`;if (toStdout) {  process.stdout.write(out);} else {  fs.writeFileSync(path.join(evalsDir, "RESULTS.md"), out);  console.log(`wrote ${path.relative(repoRoot, path.join(evalsDir, "RESULTS.md"))} from ${runs.length} run(s)`);}