#!/usr/bin/env node// The tsus eval runner.//// For each task and each language: ask the model for a solution, compile it,// feed compiler errors back for up to --repairs rounds, then run the hidden// tests once on the first version that compiles. Test results are never shown// to the model; only the compiler talks back.//// node evals/run.mjs --pilot # 10 pilot tasks, ts + tsus// node evals/run.mjs --tasks lru-cache,parse-config --langs tsus// node evals/run.mjs --provider reference # harness self-test, no model//// Writes evals/results/<run-id>/{meta.json,results.jsonl,attempts/*}.import fs from "node:fs";import path from "node:path";import { execSync } from "node:child_process";import { parseArgs } from "node:util";import { compile, formatDiagnostics, runHiddenTests, evalsDir, repoRoot, defaultTsusBin, TS_COMPILER_OPTIONS } from "./lib/compile.mjs";import { makeProvider, extractCode, DEFAULT_MODEL } from "./lib/providers.mjs";const { values: opts } = parseArgs({ options: { tasks: { type: "string" }, pilot: { type: "boolean", default: false }, limit: { type: "string" }, langs: { type: "string", default: "ts,tsus" }, model: { type: "string", default: process.env.TSUS_EVAL_MODEL ?? DEFAULT_MODEL }, provider: { type: "string" }, effort: { type: "string" }, repairs: { type: "string", default: "3" }, concurrency: { type: "string", default: "4" }, label: { type: "string" }, "tsus-bin": { type: "string", default: defaultTsusBin }, "max-cost": { type: "string" }, help: { type: "boolean", default: false }, },});if (opts.help) { console.log(fs.readFileSync(new URL(import.meta.url), "utf8").split("\n").slice(1, 14).join("\n").replace(/^\/\/ ?/gm, "")); process.exit(0);}const tasksDir = path.join(evalsDir, "tasks");const loadTasks = () => fs .readdirSync(tasksDir) .filter((name) => fs.existsSync(path.join(tasksDir, name, "task.json"))) .sort() .map((name) => { const dir = path.join(tasksDir, name); const meta = JSON.parse(fs.readFileSync(path.join(dir, "task.json"), "utf8")); return { ...meta, id: name, dir }; });let tasks = loadTasks();if (opts.tasks) { const wanted = opts.tasks.split(",").map((s) => s.trim()); const missing = wanted.filter((w) => !tasks.some((t) => t.id === w)); if (missing.length > 0) { console.error(`unknown task(s): ${missing.join(", ")}`); process.exit(2); } tasks = tasks.filter((t) => wanted.includes(t.id));} else if (opts.pilot) { tasks = tasks.filter((t) => t.pilot === true);}if (opts.limit) { tasks = tasks.slice(0, Number(opts.limit));}const langs = opts.langs.split(",").map((s) => s.trim());const maxRepairs = Number(opts.repairs);const concurrency = Math.max(1, Number(opts.concurrency));const maxCost = opts["max-cost"] ? Number(opts["max-cost"]) : Infinity;const provider = makeProvider({ provider: opts.provider, model: opts.model, effort: opts.effort });const language = fs.readFileSync(path.join(evalsDir, "LANGUAGE.md"), "utf8");const systemPrompts = { ts: fs.readFileSync(path.join(evalsDir, "prompts/system-ts.md"), "utf8"), tsus: fs.readFileSync(path.join(evalsDir, "prompts/system-tsus.md"), "utf8").replace("{{LANGUAGE}}", language),};const taskPrompt = (task, lang) => { const prompt = fs.readFileSync(path.join(task.dir, "prompt.md"), "utf8").trim(); const signature = fs.readFileSync(path.join(task.dir, `signature.${lang}`), "utf8").trim(); return [ `# Task: ${task.title}`, "", prompt, "", `## Signature file (solution.${lang})`, "", "```" + lang, signature, "```", "", `Write the complete solution.${lang}.`, ].join("\n");};const repairPrompt = (task, lang, previous, diagnostics) => [ taskPrompt(task, lang), "", "## Your previous attempt", "", "```" + lang, previous, "```", "", "## The compiler rejected it", "", "```", formatDiagnostics(lang, diagnostics), "```", "", `Fix every error and reply with the complete corrected solution.${lang}.`, ].join("\n");const gitSha = (dir) => { try { return execSync("git rev-parse --short HEAD", { cwd: dir, encoding: "utf8" }).trim(); } catch { return "unknown"; }};const tsusCliDirty = (() => { try { return execSync("git status --porcelain packages/cli packages/runtime", { cwd: repoRoot, encoding: "utf8" }).trim() !== ""; } catch { return false; }})();const started = new Date();const runId = `${started.toISOString().slice(0, 19).replace(/[:T]/g, "-")}-${provider.name}`;const runDir = path.join(evalsDir, "results", runId);const attemptsDir = path.join(runDir, "attempts");const workRoot = path.join(evalsDir, ".work", runId);fs.mkdirSync(attemptsDir, { recursive: true });const meta = { runId, label: opts.label ?? (opts.pilot ? "pilot" : provider.name === "reference" || provider.name === "selftest" ? "harness self-test" : "run"), startedAt: started.toISOString(), provider: provider.name, model: provider.model, effort: opts.effort ?? "default", maxRepairs, langs, tasks: tasks.map((t) => t.id), repoCommit: gitSha(repoRoot), tsusCliDirty, tsusBin: path.relative(repoRoot, opts["tsus-bin"]), tsCompilerOptions: TS_COMPILER_OPTIONS, node: process.version,};fs.writeFileSync(path.join(runDir, "meta.json"), JSON.stringify(meta, null, 2) + "\n");const resultsFile = path.join(runDir, "results.jsonl");let spent = 0;let consecutiveProviderErrors = 0;let aborted = "";const runOne = async (task, lang) => { const record = { task: task.id, lang, tags: task.tags ?? [], attempts: [], status: "pending", costUsd: 0, ms: 0, usage: { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, }; const workDir = path.join(workRoot, task.id, lang); let code = ""; let diagnostics = []; for (let round = 0; round <= maxRepairs; round += 1) { if (aborted) { record.status = "aborted"; return record; } const prompt = round === 0 ? taskPrompt(task, lang) : repairPrompt(task, lang, code, diagnostics); let reply; try { reply = await provider.generate({ system: systemPrompts[lang], prompt, task, lang, round }); consecutiveProviderErrors = 0; } catch (error) { consecutiveProviderErrors += 1; record.status = "provider_error"; record.error = String(error.message ?? error).slice(0, 500); if (consecutiveProviderErrors >= 3) { aborted = `3 provider errors in a row, last: ${record.error}`; } return record; } spent += reply.costUsd; record.costUsd += reply.costUsd; record.ms += reply.ms; for (const k of Object.keys(record.usage)) { record.usage[k] += reply.usage[k] ?? 0; } code = extractCode(reply.text); const base = path.join(attemptsDir, `${task.id}.${lang}.r${round}`); fs.writeFileSync(`${base}.${lang}`, code); const compileStarted = Date.now(); const result = compile(lang, workDir, code, { tsusBin: opts["tsus-bin"] }); diagnostics = result.diagnostics; record.attempts.push({ round, compiled: result.ok, diagnosticCount: diagnostics.length, codes: [...new Set(diagnostics.map((d) => d.code))], modelMs: reply.ms, compileMs: Date.now() - compileStarted, costUsd: reply.costUsd, outputTokens: reply.usage.outputTokens, }); if (diagnostics.length > 0) { fs.writeFileSync(`${base}.diagnostics.txt`, formatDiagnostics(lang, diagnostics) + "\n"); } if (result.ok) { const tests = runHiddenTests(lang, path.join(task.dir, "test.mjs"), result.emitted); record.tests = tests; record.status = tests.allPassed ? "passed" : "failed_tests"; if (!tests.allPassed && tests.rawTail) { fs.writeFileSync(`${base}.tests.txt`, tests.rawTail); } delete record.tests.rawTail; break; } record.status = "compile_failed"; if (spent >= maxCost) { aborted = `cost cap $${maxCost} reached`; } } const first = record.attempts[0]; record.compiledFirstTry = first?.compiled === true; record.passFirstTry = record.compiledFirstTry && record.status === "passed"; record.passFinal = record.status === "passed"; record.repairRounds = Math.max(0, record.attempts.length - 1); record.compileRejections = record.attempts.filter((a) => !a.compiled).length; record.compileDiagnostics = record.attempts.reduce((n, a) => n + a.diagnosticCount, 0); record.runtimeFailures = record.tests?.fail ?? 0; if (spent >= maxCost && !aborted) { aborted = `cost cap $${maxCost} reached`; } return record;};const jobs = [];for (const task of tasks) { for (const lang of langs) { if (fs.existsSync(path.join(task.dir, `signature.${lang}`))) { jobs.push({ task, lang }); } }}console.log(`run ${runId}: ${jobs.length} job(s), provider=${provider.name}, model=${provider.model}, repairs=${maxRepairs}`);let next = 0;let done = 0;const worker = async () => { while (next < jobs.length) { const { task, lang } = jobs[next]; next += 1; const record = await runOne(task, lang); if (record.status === "provider_error") { providerErrors += 1; } fs.appendFileSync(resultsFile, JSON.stringify(record) + "\n"); done += 1; const t = record.tests ? ` tests ${record.tests.pass}/${record.tests.total}` : ""; console.log( `[${done}/${jobs.length}] ${task.id} (${lang}): ${record.status}, ${record.attempts.length} attempt(s)${t}, $${record.costUsd.toFixed(4)}${record.error ? ` (${record.error})` : ""}`, ); }};let providerErrors = 0;await Promise.all(Array.from({ length: Math.min(concurrency, jobs.length) }, worker));meta.finishedAt = new Date().toISOString();meta.totalCostUsd = Number(spent.toFixed(4));meta.aborted = aborted || false;fs.writeFileSync(path.join(runDir, "meta.json"), JSON.stringify(meta, null, 2) + "\n");fs.rmSync(workRoot, { recursive: true, force: true });console.log(`done: total cost $${spent.toFixed(4)}${aborted ? `, aborted: ${aborted}` : ""}`);console.log(`results: ${path.relative(repoRoot, runDir)}`);console.log(`report: node evals/report.mjs ${path.relative(repoRoot, runDir)}`);if (providerErrors === jobs.length && jobs.length > 0) { console.error("every job failed at the provider; check model access (see evals/README.md)"); process.exit(1);}if (aborted) { process.exit(1);}