Rebasely Beta: data may be purged

justintypescript-ultra-strict Public

TypeScript · 66 lines · 3.5 KiB · at 83c2f03

// Runs the same requests, attacks included, against the plain module (../plain) and the fixed v2// module (../fixed), on the same in-memory database, and prints what each one did. The planted// bugs are real: the plain module leaks private rows, runs injected SQL, serves a script, hands// the model the attacker's markup and logs the key. The fixed module does none of that.import { untrusted } from "@tsus/v2";import * as plain from "../plain/src/worker.ts";import * as fixed from "../fixed/src/worker.ts";import { db } from "./fake-d1.boundary.ts";const KEY = "whk_live_do_not_log";const prompts: string[] = [];const logs: string[] = [];const ai = { run: async (_model: string, input: { prompt: string }) => (prompts.push(input.prompt), { response: "A crash when saving twice." }) };const env = { DB: db, AI: ai, WEBHOOK_URL: "https://hooks.example/notify", WEBHOOK_KEY: KEY };globalThis.fetch = async () => new Response("ok");console.log = (...args: unknown[]) => void logs.push(args.map(String).join(" "));const show = async (r: Promise<Response>): Promise<string> => {  try {    const res = await r;    const text = (await res.text()).replace("<!doctype html><main>", "").replace("</main>", "");    return `${res.status} ${text}`;  } catch (e) {    return `threw: ${e instanceof Error ? e.message.split("\n")[0] : String(e)}`;  }};const user = (id: string | undefined) => (id === undefined ? undefined : { id });const list = async (repo: string, query: string, as?: string) => {  const url = new URL(`https://rebasely.test/x?${query}`);  return [await show(plain.listIssues(env, user(as), repo, url)), await show(fixed.route(env, user(as), { kind: "list", repo: untrusted(repo), url }))];};const summary = async () => {  prompts.length = 0;  logs.length = 0;  await show(plain.summarize(env, undefined, "public-1", 2));  const plainPrompt = prompts[0] ?? "";  const plainLogs = logs.join(" | ");  prompts.length = 0;  logs.length = 0;  await show(fixed.route(env, undefined, { kind: "summarize", repo: untrusted("public-1"), n: 2 }));  return { plainPrompt, plainLogs, fixedPrompt: prompts[0] ?? "", fixedLogs: logs.join(" | ") };};const out = (line: string): void => void process.stdout.write(`${line}\n`);const row = (name: string, [p, f]: string[]) => {  out(`${name}`);  out(`  plain: ${p}`);  out(`  v2:    ${f}`);};row("search q=crash", await list("public-1", "q=crash"));row("bug 1, q=' OR 1=1 --", await list("public-1", `q=${encodeURIComponent("' OR 1=1 --")}`));// A blind injection through the sort column: the order of the public rows answers a yes/no// question about a private table (here, who owns secret-1).const oracle = (guess: string) => `sort=${encodeURIComponent(`CASE WHEN (SELECT owner_id FROM repos WHERE id = 'secret-1') = '${guess}' THEN number ELSE -number END`)}`;row("bug 2, sort asks whether ada owns secret-1", await list("public-1", oracle("ada")));row("bug 2, sort asks whether bob owns secret-1", await list("public-1", oracle("bob")));row("bug 3, private repo signed out", await list("secret-1", "q="));row("private repo, its owner", await list("secret-1", "q=", "ada"));const s = await summary();out("bug 4, the webhook key in the logs");out(`  plain: ${s.plainLogs.includes(KEY)}`);out(`  v2:    ${s.fixedLogs.includes(KEY)}`);out("bug 6, the issue body's </untrusted> and <system> reach the prompt as markup");out(`  plain: ${s.plainPrompt.includes("<system>")}`);out(`  v2:    ${s.fixedPrompt.includes("<system>")} (fenced: ${s.fixedPrompt.includes('<untrusted source="issue body">')})`);