// Model providers. Each provider exposes// generate({ system, prompt, task, lang, round }) -> { text, usage, costUsd, ms }//// claude-cli `claude -p` headless (default when no ANTHROPIC_API_KEY)// anthropic Messages API through @anthropic-ai/sdk (default when ANTHROPIC_API_KEY is set)// reference returns the task's reference solution (harness self-test, no model)// selftest round 0 returns the reference with a planted compile error, round 1// the clean reference; proves the repair loop end to end (no model)//// Keys are never logged: the CLI and the SDK read credentials themselves.import fs from "node:fs";import os from "node:os";import path from "node:path";import { spawn } from "node:child_process";// Price per million tokens, for providers that do not report cost themselves.const PRICES = { "claude-sonnet-5-5": { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5 }, "claude-sonnet-5": { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5 }, "claude-sonnet-4-6": { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 }, "claude-opus-5-5": { input: 4, output: 20, cacheRead: 0.2, cacheWrite: 5 }, "claude-haiku-5-5": { input: 0.1, output: 0.5, cacheRead: 0.01, cacheWrite: 0.125 },};export const DEFAULT_MODEL = "claude-sonnet-5-5";const emptyUsage = () => ({ inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 });const priceUsage = (model, usage) => { const p = PRICES[model]; if (!p) { return 0; } return ( (usage.inputTokens * p.input + usage.outputTokens * p.output + usage.cacheReadTokens * p.cacheRead + usage.cacheWriteTokens * p.cacheWrite) / 1_000_000 );};// Environment variables that tie a child `claude` process to a parent Claude// Code session. Inherited, they make the child try to borrow the parent's auth// and fail; the eval wants a standalone, stateless call.const NESTED_SESSION_VARS = [ "CLAUDECODE", "CLAUDE_CODE_ENTRYPOINT", "CLAUDE_CODE_CHILD_SESSION", "CLAUDE_CODE_SESSION_ID", "CLAUDE_CODE_HOST_SESSION_ID", "CLAUDE_CODE_SDK_HAS_HOST_AUTH_REFRESH", "CLAUDE_CODE_MESSAGING_SOCKET", "CLAUDE_CODE_MESSAGING_TOKEN", "CLAUDE_CODE_SIMPLE", "CLAUDE_AGENT_SDK_VERSION",];const claudeCli = ({ model, timeoutMs = 600_000 }) => ({ name: "claude-cli", model, generate: ({ system, prompt }) => new Promise((resolve, reject) => { const env = { ...process.env }; for (const key of NESTED_SESSION_VARS) { delete env[key]; } const started = Date.now(); const args = [ "-p", "--model", model, "--system-prompt", system, "--tools", "", "--strict-mcp-config", "--no-session-persistence", "--output-format", "json", ]; // Run from an empty temp dir so no project CLAUDE.md leaks into context. const cwd = fs.mkdtempSync(path.join(os.tmpdir(), "tsus-eval-")); const child = spawn(process.env.CLAUDE_BIN ?? "claude", args, { cwd, env, stdio: ["pipe", "pipe", "pipe"] }); let stdout = ""; let stderr = ""; const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs); child.stdout.on("data", (d) => { stdout += d; }); child.stderr.on("data", (d) => { stderr += d; }); child.on("error", reject); child.on("close", () => { clearTimeout(timer); fs.rmSync(cwd, { recursive: true, force: true }); let data; try { data = JSON.parse(stdout); } catch { reject(new Error(`claude -p returned non-JSON output: ${(stdout || stderr).slice(0, 500)}`)); return; } if (data.is_error) { reject(new Error(`claude -p failed: ${String(data.result).slice(0, 500)}`)); return; } const u = data.usage ?? {}; resolve({ text: data.result ?? "", usage: { inputTokens: u.input_tokens ?? 0, outputTokens: u.output_tokens ?? 0, cacheReadTokens: u.cache_read_input_tokens ?? 0, cacheWriteTokens: u.cache_creation_input_tokens ?? 0, }, costUsd: data.total_cost_usd ?? 0, ms: Date.now() - started, servedBy: Object.keys(data.modelUsage ?? {}).join(",") || model, }); }); child.stdin.end(prompt); }),});const anthropicApi = ({ model, effort }) => { let clientPromise; const getClient = () => { clientPromise ??= import("@anthropic-ai/sdk") .catch(() => { throw new Error("the anthropic provider needs @anthropic-ai/sdk: run `npm install --prefix evals`"); }) .then((mod) => new mod.default()); return clientPromise; }; return { name: "anthropic", model, generate: async ({ system, prompt }) => { const client = await getClient(); const started = Date.now(); const response = await client.messages.create({ model, max_tokens: 16000, system, messages: [{ role: "user", content: prompt }], ...(effort ? { output_config: { effort } } : {}), }); if (response.stop_reason === "refusal") { throw new Error(`model refused (${response.stop_details?.category ?? "no category"})`); } const text = response.content .filter((b) => b.type === "text") .map((b) => b.text) .join("\n"); const usage = { inputTokens: response.usage.input_tokens ?? 0, outputTokens: response.usage.output_tokens ?? 0, cacheReadTokens: response.usage.cache_read_input_tokens ?? 0, cacheWriteTokens: response.usage.cache_creation_input_tokens ?? 0, }; return { text, usage, costUsd: priceUsage(model, usage), ms: Date.now() - started, servedBy: response.model }; }, };};const fence = (lang, code) => "```" + lang + "\n" + code + "\n```\n";// A compile error each checker is guaranteed to reject.const PLANTED_ERROR = { ts: 'const __planted: number = "not a number";\nvoid __planted;\n', tsus: "let __planted = null;\nvoid __planted;\n",};const referenceProvider = ({ broken }) => ({ name: broken ? "selftest" : "reference", model: "none", generate: async ({ task, lang, round }) => { const code = fs.readFileSync(path.join(task.dir, `reference.${lang}`), "utf8"); const text = broken && round === 0 ? code + "\n" + PLANTED_ERROR[lang] : code; return { text: fence(lang, text), usage: emptyUsage(), costUsd: 0, ms: 0, servedBy: "none" }; },});export const makeProvider = ({ provider, model, effort }) => { const chosen = provider ?? (process.env.ANTHROPIC_API_KEY ? "anthropic" : "claude-cli"); switch (chosen) { case "claude-cli": return claudeCli({ model }); case "anthropic": return anthropicApi({ model, effort }); case "reference": return referenceProvider({ broken: false }); case "selftest": return referenceProvider({ broken: true }); default: throw new Error(`unknown provider ${chosen}`); }};// Pull the solution out of a reply: the longest fenced block wins, since models// sometimes show a snippet before the full file.export const extractCode = (text) => { const blocks = [...text.matchAll(/```[a-zA-Z]*\n([\s\S]*?)```/g)].map((m) => m[1]); if (blocks.length === 0) { return text.trim(); } return blocks.reduce((a, b) => (b.length >= a.length ? b : a));};