Rebasely Beta: data may be purged

justintypescript-ultra-strict Public

JavaScript · 223 lines · 12.5 KiB · at 83c2f03

import { __tsus_match_lit, __tsus_match } from "./_runtime/index.js"; // The on-call watcher's decision rules, in TypeScript-Ultra-Strict (tsus; docs/v2/tsus-pilot.md):// which events are actionable and how urgent, dedupe per incident, the queue's order and cap, the// hourly rate limit, the skip and escalation rules after a run, the API outage run and the morning// digest's schedule and fallback text. Everything here is pure: lib/watcher.ts (plain .ts) does the// I/O, holds the state, and converts at the boundary (null in the API's and the state file's types// is read through fromNullable here; Option comes back and the .ts side turns it into null).//// Built by `npm run tsus:build` into ops/oncall/dist (committed: the watcher runs it with plain node).import { Some, None } from "./_runtime/index.js";import { field, fromNullable, isRecord, wrapSync } from "./_runtime/boundary.js";export const DEFAULT_CONFIG = {    environments: ["production"],    pollMs: 20_000,    heartbeatMs: 60_000,    runTimeoutMs: 15 * 60_000,    maxRunsPerHour: 12,    dedupeHours: 6,    apiDownMs: 3 * 60_000,    digest: true,    digestHour: 9,    digestTz: "America/Los_Angeles",    queueMax: 50,};const HOUR = 3_600_000;const DAY = 24 * HOUR;// Alerts that never start a run: user feedback (bug reports, a reporter's reply, bursts of reports,// the daily user-facing issue digest). Their text comes from users and is untrusted, and the owner's// rule is that triage of it never makes an agent write code: they are logged and reach the morning// digest, which acts on nothing (DIGEST.md). The owner or an agent the owner starts reads them with// scripts/bug-reports.ts (.rebasely/knowledge/guides/bug-triage.md).export const TRIAGE_ONLY_PREFIXES = ["bug_report:", "bug_report_reply:", "bug_reports:", "ux_issues:"];// --- classification and severity ---// The queue order of an alert class: critical first, then pages, then tickets. An unknown class// ranks as a ticket.export const rankOf = (cls) => __tsus_match_lit(cls)([    ["critical", () => (0)],    ["page", () => (1)],    ["ticket", () => (2)],    ["direct", () => (3)],], () => (2));const parseJson = wrapSync(JSON.parse);// An event's data column: a JSON object, or {} for anything else.export const eventData = (raw) => __tsus_match(raw)({    Some: (text) => ((text === "" ? {} : parseJson(text).ok().andThen(recordOf).unwrapOr({}))),    None: () => (({})),});const recordOf = (v) => (isRecord(v) ? Some(v) : None);// The API's columns are null or a string; an empty string counts as absent, as it always has.const present = (s) => s.andThen((v) => (v === "" ? None : Some(v)));// An event without an environment is not filtered out.const watched = (environment, environments) => __tsus_match(present(environment))({    Some: (env) => (environments.includes(env)),    None: () => (true),});// The on-call system's own alerts (the agent's tickets, the watcher-offline page) are not runs.// Drills are.const ownAlert = (key) => key.startsWith("oncall:") && !key.startsWith("oncall:drill:");const triageOnly = (key) => TRIAGE_ONLY_PREFIXES.some((p) => key.startsWith(p));const alertJob = (e, key, data) => {    if (ownAlert(key) || triageOnly(key))        return None;    const cls = typeof data.class === "string" ? data.class : "ticket";    if (cls === "direct")        return None;    return Some({ key, alertKey: Some(key), incident: e.kind === "alert_opened" ? Some(e.at) : None, rank: rankOf(cls), data });};// The org's suspension ticket is abuse:<orgId>: the flag and the ticket are one incident.const abuseJob = (e, data) => {    const subject = typeof data.subject_id === "string" ? data.subject_id : String(e.id);    return { key: `abuse:${subject}`, alertKey: None, incident: None, rank: 2, data };};// Which events start a run: an alert opened or whose severity rose, and an abuse flag, in the// watched environments.export const classify = (e, environments) => {    if (!watched(fromNullable(e.environment), environments))        return None;    const data = eventData(fromNullable(e.data));    const key = present(fromNullable(e.key));    if ((e.kind === "alert_opened" || e.kind === "alert_severity_rose") && key.isSome())        return alertJob(e, key.value, data);    if (e.kind === "abuse_flag")        return Some(abuseJob(e, data));    return None;};// The first poll picks up the open alerts nobody has acknowledged.export const pickUpOpenAlert = (a, environments) => present(fromNullable(a.ack_by)).isNone() && environments.includes(a.environment);// --- dedupe and the queue ---// A job without an incident time, or a handled run without one, is the same incident; otherwise an// incident that opened after the key was handled is a new one.const sameIncident = (incident, handled) => __tsus_match(incident)({    Some: (opened) => (fromNullable(handled.incident).isNone() || opened <= handled.at),    None: () => (true),});// Whether a key handled within dedupeHours makes this job a repeat of the same incident.export const isDuplicate = (handled, incident, now, dedupeHours) => __tsus_match(handled)({    Some: (h) => (now - h.at < dedupeHours * HOUR && sameIncident(incident, h)),    None: () => (false),});export const alreadyQueued = (key, queued, running) => {    const isRunning = __tsus_match(running)({        Some: (j) => (j.key === key),        None: () => (false),    });    return isRunning || queued.some((q) => q.key === key);};export const queueFull = (length, queueMax) => length >= queueMax;export const queueFullMessage = (queueMax, key) => `The on-call agent's queue is full (${queueMax} events); it did not look at ${key}.`;// Critical first, then pages, then the rest, oldest first.export const byPriority = (a, b) => a.rank - b.rank || a.queuedAt - b.queuedAt;// Handled keys are kept for 7 days.export const pruneHandled = (handled, now) => {    const cutoff = now - 7 * DAY;    return Object.fromEntries(Object.entries(handled).filter(([, v]) => !(v.at < cutoff)));};// --- the rate limit ---export const recentRuns = (runs, now) => runs.filter((t) => now - t < HOUR);// Past maxRunsPerHour runs in the last hour an incident is not run: a ticket waits for the digest,// anything else reaches the owner. The digest is not counted and never limited.export const rateDecision = (job, runsLastHour, maxRunsPerHour) => {    if (job.kind === "digest" || runsLastHour < maxRunsPerHour)        return "run";    return present(fromNullable(job.alertKey)).isSome() && job.rank >= 2 ? "wait_for_digest" : "escalate";};export const rateLimitMessage = (maxRunsPerHour, title) => `The on-call agent reached its limit of ${maxRunsPerHour} runs an hour, so nobody is looking at this: ${title}`;// The alert's title from the event, else the job's key.export const jobTitle = (job) => __tsus_match(fromNullable(job.event).andThen((e) => field(e, "data")).andThen((d) => field(d, "title")))({    Some: (t) => ((typeof t === "string" ? t : job.key)),    None: () => (job.key),});// --- skip and escalation rules around a run ---// An alert that resolved before its run starts is not run, unless it was critical.export const skipResolved = (job, alert) => {    const resolved = __tsus_match(alert)({        Some: (a) => (a.state === "resolved"),        None: () => (false),    });    return resolved && job.kind === "incident" && job.rank > 0;};// The API unreachable for apiDownMs starts one run about it.export const outageRunDue = (alreadyRun, downSince, now, apiDownMs) => !alreadyRun && now - downSince >= apiDownMs;// Stopped by the watcher's own shutdown (launchctl, uninstall): not the agent's failure. An open// page still escalates on the server's timer.export const stoppedByShutdown = (stopped, out) => stopped && !out.timedOut;const failure = (out, runId, timeoutMs) => {    if (out.timedOut)        return Some(`The on-call agent did not finish within ${Math.round(timeoutMs / 60_000)} min (run ${runId}), so it was stopped.`);    if (out.exitCode !== 0) {        const code = fromNullable(out.exitCode).map((c) => String(c)).unwrapOr(fromNullable(out.signal).unwrapOr("?"));        return Some(`The on-call agent's run ${runId} failed (exit ${code}).`);    }    return None;};// Claude posted its summary on the alert during this run (act.ts note).const reportedSince = (a, startedAt) => fromNullable(a.note_by).unwrapOr("") === "claude" && fromNullable(a.note_at).unwrapOr(0) >= startedAt;// After a run: a timeout, a crash (non-zero exit), or a run that ended without its summary on an// open alert is a failure. The run's final message goes on the alert as a note, so the owner and// the digest see it even when the agent could not post it. A ticket is never escalated for it (the// policy: tickets are Claude's and the digest's; only Claude itself may escalate one); pages,// critical alerts and alert-less jobs (abuse flags, the API outage) are.// `alert` is the alert as it is after the run (None: no alert key, or the API did not answer).export const verdict = (job, out, alert, startedAt, runId, timeoutMs) => {    const alertKey = present(fromNullable(job.alertKey));    const why = failure(out, runId, timeoutMs);    if (why.isNone()) {        if (alertKey.isNone())            return { tag: "Fine" };        const settled = __tsus_match(alert)({            Some: (a) => (reportedSince(a, startedAt) || a.state !== "open"),            None: () => (true),        });        if (settled)            return { tag: "Fine" };    }    const where = out.logPath ? ` Log: ${out.logPath}` : "";    const message = `${why.unwrapOr(`The on-call agent's run ${runId} ended without posting its summary to the alert.`)}${where}`;    const found = out.result.trim();    const note = alertKey.isSome() && alert.isSome() ? Some(`${message}${found ? `\nThe run's final message: ${found}` : ""}`.slice(0, 2000)) : None;    const fallbackClass = job.rank >= 2 && alertKey.isSome() ? "ticket" : "";    const cls = __tsus_match(alert)({        Some: (a) => (fromNullable(a.class).unwrapOr(fallbackClass)),        None: () => (fallbackClass),    });    return { tag: "Failed", message, note, escalate: cls !== "ticket" && cls !== "direct" };};// --- the morning digest ---// The local date and hour in a time zone (the digest's 09:00 America/Los_Angeles).export const localParts = (t, tz) => {    const fmt = new Intl.DateTimeFormat("en-CA", { timeZone: tz, year: "numeric", month: "2-digit", day: "2-digit", hour: "2-digit", hourCycle: "h23" });    const p = Object.fromEntries(fmt.formatToParts(new Date(t)).map((x) => [x.type, x.value]));    return { day: `${p.year}-${p.month}-${p.day}`, hour: Number(p.hour) };};// Due from digestHour until 12 hours later, once per local day: a Mac that slept through 09:00 sends// it when it wakes, but not in the evening.export const digestDue = (now, last, cfg) => {    if (!cfg.digest)        return None;    const { day, hour } = localParts(now, cfg.digestTz);    const sentToday = __tsus_match(last)({        Some: (d) => (d === day),        None: () => (false),    });    return hour >= cfg.digestHour && hour < cfg.digestHour + 12 && !sentToday ? Some(day) : None;};// The digest Claude wrote, when the run succeeded with more than a line.export const digestText = (out) => out.exitCode === 0 && !out.timedOut && out.result.trim().length > 20 ? Some(out.result.trim()) : None;const list = (v) => (Array.isArray(v) ? v : []);const record = (v) => (isRecord(v) ? v : {});const prop = (v, k) => (isRecord(v) ? String(v[k]) : "undefined");// A plain digest from the API's summary, for when the Claude run cannot write one.export const fallbackDigest = (s, day) => {    const open = list(s.open);    const opened = list(s.opened);    const events = record(s.events);    const count = (k) => String(fromNullable(events[k]).unwrapOr(0));    const accounts = Object.entries(record(s.counts)).map(([env, c]) => `Accounts (${env}): ${Object.entries(record(c)).filter(([k]) => k !== "at").map(([k, v]) => `${k} ${String(v)}`).join(", ")}`);    return [        `Rebasely, last 24 hours to ${day} (the on-call agent could not write the digest; these are the API's numbers).`,        ...accounts,        `Alerts opened or resolved: ${opened.length}${opened.length ? ` (${opened.slice(0, 8).map((a) => `${prop(a, "key")} ${prop(a, "state")}`).join("; ")})` : ""}`,        `Open now: ${open.length}${open.length ? ` (${open.slice(0, 8).map((a) => prop(a, "key")).join(", ")})` : ""}`,        `Escalations: ${count("escalated")}. Abuse flags: ${count("abuse_flag")}. Milestones: ${count("milestone")}.`,    ].join("\n");};