monitor

 1@@ -7,6 +7,9 @@ Health-endpoint monitor on Cloudflare Workers + Durable Objects. See [DESIGN.md]
 2   SQLite (a trigger drops rows older than 14 days).
 3 - Alerts go to `ALERT_TO` when an endpoint starts failing in a region, every `REMINDER_HOURS`
 4   while it keeps failing, and when it recovers. Anything other than HTTP 200 (including timeouts) is a failure.
 5+- Each alert also goes to Pushover once its secrets are set: a second, independent channel. Each
 6+  channel tracks its own last alert, so a failed send is retried on that channel only. Pushover
 7+  messages list the failing entries of a `/healthz` `checks` object; recoveries arrive at priority -1 (quiet).
 8 - An hourly cron (and any homepage visit) makes sure every region's alarm loop is running.
 9 - `https://monitor.rtw.run/` shows the latest result per endpoint; `/endpoint?url=…` shows history.
10 
11@@ -20,6 +23,13 @@ npx wrangler secret put AUTH_USER
12 npx wrangler secret put AUTH_PASS
13 ```
14 
15+Pushover is off until both of its secrets are set (an application token and your user key):
16+
17+```sh
18+npx wrangler secret put PUSHOVER_TOKEN
19+npx wrangler secret put PUSHOVER_USER
20+```
21+
22 ## Development
23 
24 ```sh

  1@@ -1,5 +1,6 @@
  2 import { DurableObject } from "cloudflare:workers";
  3 import { sendAlert, type AlertKind } from "./email";
  4+import { pushoverEnabled, sendPushover } from "./pushover";
  5 
  6 export const REGIONS = ["wnam", "enam", "weur", "eeur", "apac"] as const;
  7 export type Region = (typeof REGIONS)[number];
  8@@ -33,7 +34,18 @@ const RETENTION_MS = 14 * 24 * 60 * 60 * 1000;
  9 const MAX_BODY_CHARS = 64 * 1024;
 10 
 11 type CheckRow = Omit<Check, "region">;
 12-type AlertRow = { endpoint: string; failing_since: number | null; last_alert_ts: number | null };
 13+type AlertRow = {
 14+  endpoint: string;
 15+  failing_since: number | null;
 16+  last_alert_ts: number | null;
 17+  last_pushover_ts: number | null;
 18+};
 19+
 20+/** The alert channels. Each keeps its own last-sent time, so one failing never holds up the other. */
 21+const CHANNELS = [
 22+  { column: "last_alert_ts", enabled: (_env: Env) => true, send: sendAlert },
 23+  { column: "last_pushover_ts", enabled: pushoverEnabled, send: sendPushover },
 24+] as const;
 25 
 26 export class MonitorDO extends DurableObject<Env> {
 27   private sql: SqlStorage;
 28@@ -62,6 +74,10 @@ export class MonitorDO extends DurableObject<Env> {
 29         last_alert_ts INTEGER
 30       );
 31     `);
 32+    const columns = this.sql.exec<{ name: string }>(`SELECT name FROM pragma_table_info('alert_state')`).toArray();
 33+    if (!columns.some((c) => c.name === "last_pushover_ts")) {
 34+      this.sql.exec(`ALTER TABLE alert_state ADD COLUMN last_pushover_ts INTEGER`);
 35+    }
 36   }
 37 
 38   /** The region this instance monitors from; recorded by ensureAlarm(). */
 39@@ -138,37 +154,40 @@ export class MonitorDO extends DurableObject<Env> {
 40   }
 41 
 42   /**
 43-   * Emails on healthy→failing, every REMINDER_HOURS while still failing, and on recovery.
 44-   * A failed send leaves last_alert_ts unchanged so the next check retries it.
 45+   * Alerts on every enabled channel (email; Pushover once its secrets are set) on healthy→failing,
 46+   * every REMINDER_HOURS while still failing, and on recovery. A failed send leaves that channel's
 47+   * last-sent time unchanged so the next check retries it, on that channel only.
 48    */
 49   private async updateAlertState(check: Check): Promise<void> {
 50     const state = this.sql
 51       .exec<AlertRow>(`SELECT * FROM alert_state WHERE endpoint = ?`, check.endpoint)
 52-      .toArray()[0] ?? { endpoint: check.endpoint, failing_since: null, last_alert_ts: null };
 53+      .toArray()[0] ?? { endpoint: check.endpoint, failing_since: null, last_alert_ts: null, last_pushover_ts: null };
 54     const reminderMs = Number(this.env.REMINDER_HOURS) * 60 * 60 * 1000;
 55-
 56-    let kind: AlertKind | null = null;
 57-    if (check.status !== 200) {
 58-      if (state.failing_since === null) state.failing_since = check.ts;
 59-      if (state.last_alert_ts === null) kind = "down";
 60-      else if (check.ts - state.last_alert_ts >= reminderMs) kind = "still down";
 61-    } else if (state.failing_since !== null) {
 62-      kind = "recovered";
 63-    }
 64-
 65-    if (kind) {
 66-      const sent = await sendAlert(this.env, kind, check, state.failing_since);
 67-      if (kind === "recovered") {
 68-        state.failing_since = null;
 69-        state.last_alert_ts = null;
 70-      } else if (sent) {
 71-        state.last_alert_ts = check.ts;
 72-      }
 73+    const failing = check.status !== 200;
 74+    const recovered = !failing && state.failing_since !== null;
 75+    if (failing && state.failing_since === null) state.failing_since = check.ts;
 76+
 77+    await Promise.all(
 78+      CHANNELS.filter((ch) => ch.enabled(this.env)).map(async (ch) => {
 79+        const last = state[ch.column];
 80+        let kind: AlertKind | null = null;
 81+        if (recovered) kind = "recovered";
 82+        else if (failing && last === null) kind = "down";
 83+        else if (failing && last !== null && check.ts - last >= reminderMs) kind = "still down";
 84+        if (!kind) return;
 85+        const sent = await ch.send(this.env, kind, check, state.failing_since);
 86+        if (sent && kind !== "recovered") state[ch.column] = check.ts;
 87+      }),
 88+    );
 89+    if (recovered) {
 90+      state.failing_since = null;
 91+      state.last_alert_ts = null;
 92+      state.last_pushover_ts = null;
 93     }
 94 
 95     this.sql.exec(
 96-      `INSERT OR REPLACE INTO alert_state (endpoint, failing_since, last_alert_ts) VALUES (?, ?, ?)`,
 97-      check.endpoint, state.failing_since, state.last_alert_ts,
 98+      `INSERT OR REPLACE INTO alert_state (endpoint, failing_since, last_alert_ts, last_pushover_ts) VALUES (?, ?, ?, ?)`,
 99+      check.endpoint, state.failing_since, state.last_alert_ts, state.last_pushover_ts,
100     );
101   }
102 }

 1@@ -0,0 +1,71 @@
 2+import type { AlertKind } from "./email";
 3+import type { Check } from "./monitor";
 4+
 5+const API = "https://api.pushover.net/1/messages.json";
 6+/** Pushover's limits: title 250 characters, message 1024. */
 7+const MAX_TITLE = 250;
 8+const MAX_MESSAGE = 1024;
 9+
10+/** Optional secrets; the channel is off until both are set. */
11+type PushoverEnv = Env & { PUSHOVER_TOKEN?: string; PUSHOVER_USER?: string };
12+
13+export function pushoverEnabled(env: Env): boolean {
14+  const e = env as PushoverEnv;
15+  return Boolean(e.PUSHOVER_TOKEN && e.PUSHOVER_USER);
16+}
17+
18+/**
19+ * Sends an alert as a Pushover notification, the second channel next to email. Returns false (and
20+ * logs) on failure rather than throwing. Call only when pushoverEnabled().
21+ */
22+export async function sendPushover(env: Env, kind: AlertKind, check: Check, failingSince: number | null): Promise<boolean> {
23+  const e = env as PushoverEnv;
24+  const host = check.endpoint.replace(/^https?:\/\//, "");
25+  const result = check.status ? `HTTP ${check.status}` : (check.error ?? "no response");
26+  const title = `${kind.toUpperCase()}: ${host} (${result}) from ${check.region}`;
27+  const lines: string[] = [];
28+  if (failingSince !== null) lines.push(`Failing since ${new Date(failingSince).toISOString()}`);
29+  if (kind !== "recovered") lines.push(...failureLines(check));
30+  const form = new URLSearchParams({
31+    token: e.PUSHOVER_TOKEN!,
32+    user: e.PUSHOVER_USER!,
33+    title: title.slice(0, MAX_TITLE),
34+    message: (lines.join("\n") || result).slice(0, MAX_MESSAGE),
35+    // Recoveries arrive quietly; failures make a sound.
36+    priority: kind === "recovered" ? "-1" : "0",
37+    url: `${env.PUBLIC_URL}/endpoint?url=${encodeURIComponent(check.endpoint)}`,
38+    url_title: "History",
39+    timestamp: String(Math.floor(check.ts / 1000)),
40+  });
41+  try {
42+    const res = await fetch(API, { method: "POST", body: form, signal: AbortSignal.timeout(Number(env.TIMEOUT_MS)) });
43+    const reply = await res.text();
44+    if (!res.ok || !reply.includes('"status":1')) throw new Error(`HTTP ${res.status}: ${reply.slice(0, 200)}`);
45+    console.log("alert pushover sent", { endpoint: check.endpoint, kind, title });
46+    return true;
47+  } catch (err) {
48+    console.error("alert pushover failed", { endpoint: check.endpoint, kind, error: String(err) });
49+    return false;
50+  }
51+}
52+
53+/**
54+ * What failed, for the message body: the failing entries of a /healthz `checks` object
55+ * ({name: {status, error?, detail?}}), else the raw body or the fetch error.
56+ */
57+export function failureLines(check: Check): string[] {
58+  if (check.error && !check.status) return [check.error];
59+  if (!check.body) return [];
60+  try {
61+    const checks = (JSON.parse(check.body) as { checks?: Record<string, { status?: string; error?: string; detail?: string }> }).checks;
62+    if (checks && typeof checks === "object") {
63+      const failing = Object.entries(checks)
64+        .filter(([, c]) => c?.status !== "ok")
65+        .map(([name, c]) => `${name}: ${c?.error ?? c?.detail ?? c?.status ?? "failed"}`);
66+      if (failing.length) return failing;
67+    }
68+  } catch {
69+    // Not JSON: fall through to the raw body.
70+  }
71+  return [check.body];
72+}

  1@@ -19,20 +19,38 @@ async function freshMonitor(region: Region): Promise<DurableObjectStub<MonitorDO
  2   return stub;
  3 }
  4 
  5-/** Routes outbound fetches: each endpoint maps to a status code, or an Error to throw. */
  6-function mockEndpoints(responses: Record<string, number | Error>) {
  7-  return vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => {
  8+const PUSHOVER = "https://api.pushover.net/1/messages.json";
  9+
 10+/**
 11+ * Routes outbound fetches: each endpoint maps to a status code, or an Error to throw. Pushover
 12+ * posts are recorded in `pushes`; they succeed unless `pushoverFails` is set.
 13+ */
 14+function mockEndpoints(responses: Record<string, number | Error>, pushes: URLSearchParams[] = [], pushoverFails = false) {
 15+  return vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
 16     const url = input instanceof Request ? input.url : String(input);
 17+    if (url === PUSHOVER) {
 18+      if (pushoverFails) return new Response('{"status":0,"errors":["down"]}', { status: 500 });
 19+      pushes.push(new URLSearchParams(String(init?.body)));
 20+      return new Response('{"status":1,"request":"x"}');
 21+    }
 22     const r = responses[url];
 23     if (r instanceof Error) throw r;
 24     return new Response(JSON.stringify({ ok: r === 200, url }), { status: r ?? 404 });
 25   });
 26 }
 27 
 28+/** Turns the Pushover channel on for this monitor by giving its env the two secrets. */
 29+async function enablePushover(stub: DurableObjectStub<MonitorDO>) {
 30+  await runInDurableObject(stub, (instance: MonitorDO) => {
 31+    Object.assign((instance as unknown as { env: Env }).env, { PUSHOVER_TOKEN: "tok", PUSHOVER_USER: "usr" });
 32+  });
 33+}
 34+
 35 /** Runs one alarm cycle and returns the subjects of emails sent during it. */
 36 async function runCycle(stub: DurableObjectStub<MonitorDO>, send?: (msg: EmailMessage) => Promise<unknown>) {
 37   const subjects: string[] = [];
 38   const log = vi.spyOn(console, "log").mockImplementation(() => {});
 39+  const error = vi.spyOn(console, "error").mockImplementation(() => {});
 40   await runInDurableObject(stub, (instance: MonitorDO) => {
 41     const emailEnv = (instance as unknown as { env: Env }).env;
 42     vi.spyOn(emailEnv.EMAIL, "send").mockImplementation(async (msg: any) => {
 43@@ -44,6 +62,7 @@ async function runCycle(stub: DurableObjectStub<MonitorDO>, send?: (msg: EmailMe
 44   expect(await runDurableObjectAlarm(stub)).toBe(true);
 45   for (const [message, data] of log.mock.calls) if (message === "alert email sent") subjects.push(data.subject);
 46   log.mockRestore();
 47+  error.mockRestore();
 48   return subjects;
 49 }
 50 
 51@@ -121,6 +140,84 @@ describe("alerts", () => {
 52   });
 53 });
 54 
 55+describe("pushover", () => {
 56+  const titles = (pushes: URLSearchParams[]) => pushes.map((p) => p.get("title"));
 57+
 58+  it("stays off until both secrets are set", async () => {
 59+    const stub = await freshMonitor("weur");
 60+    const pushes: URLSearchParams[] = [];
 61+    mockEndpoints({ [A]: 503, [B]: 200 }, pushes);
 62+    expect(await runCycle(stub)).toHaveLength(1);
 63+    expect(pushes).toEqual([]);
 64+  });
 65+
 66+  it("notifies alongside email: down with the failing checks, reminder, recovery", async () => {
 67+    const stub = await freshMonitor("enam");
 68+    await enablePushover(stub);
 69+    const pushes: URLSearchParams[] = [];
 70+    const body = JSON.stringify({
 71+      status: "degraded",
 72+      checks: { disk: { status: "ok", detail: "used=18%" }, backup: { status: "fail", detail: "age=27h", error: "backup stale" } },
 73+    });
 74+    vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
 75+      const url = input instanceof Request ? input.url : String(input);
 76+      if (url === PUSHOVER) {
 77+        pushes.push(new URLSearchParams(String(init?.body)));
 78+        return new Response('{"status":1}');
 79+      }
 80+      return new Response(url === A ? body : "{}", { status: url === A ? 503 : 200 });
 81+    });
 82+
 83+    expect(await runCycle(stub)).toHaveLength(1);
 84+    expect(pushes).toHaveLength(1);
 85+    expect(pushes[0].get("title")).toBe("DOWN: a.test/healthz (HTTP 503) from enam");
 86+    expect(pushes[0].get("message")).toMatch(/^Failing since .*\nbackup: backup stale$/);
 87+    expect(pushes[0].get("priority")).toBe("0");
 88+    expect(pushes[0].get("token")).toBe("tok");
 89+    expect(pushes[0].get("user")).toBe("usr");
 90+    expect(pushes[0].get("url")).toBe(`https://monitor.rtw.run/endpoint?url=${encodeURIComponent(A)}`);
 91+
 92+    await runCycle(stub);
 93+    expect(pushes).toHaveLength(1);
 94+
 95+    await sql(stub, "UPDATE alert_state SET last_alert_ts = last_alert_ts - ?, last_pushover_ts = last_pushover_ts - ? WHERE endpoint = ?",
 96+      7 * 3600 * 1000, 7 * 3600 * 1000, A);
 97+    await runCycle(stub);
 98+    expect(titles(pushes).at(-1)).toMatch(/^STILL DOWN: a\.test/);
 99+
100+    vi.restoreAllMocks();
101+    mockEndpoints({ [A]: 200, [B]: 200 }, pushes);
102+    expect(await runCycle(stub)).toEqual([expect.stringContaining("RECOVERED")]);
103+    expect(titles(pushes).at(-1)).toBe("RECOVERED: a.test/healthz (HTTP 200) from enam");
104+    expect(pushes.at(-1)!.get("priority")).toBe("-1");
105+    expect(pushes).toHaveLength(3);
106+  });
107+
108+  it("retries only the channel that failed", async () => {
109+    // Email down: Pushover goes out once, email is retried until it works.
110+    const stub = await freshMonitor("wnam");
111+    await enablePushover(stub);
112+    const pushes: URLSearchParams[] = [];
113+    mockEndpoints({ [A]: 500, [B]: 200 }, pushes);
114+    expect(await runCycle(stub, async () => { throw new Error("email service unavailable"); })).toEqual([]);
115+    expect(titles(pushes)).toEqual([expect.stringMatching(/^DOWN: a\.test/)]);
116+    expect(await runCycle(stub)).toEqual([expect.stringContaining("DOWN: a.test/healthz")]);
117+    expect(pushes).toHaveLength(1);
118+
119+    // Pushover down: email goes out once, Pushover is retried until it works.
120+    const other = await freshMonitor("apac");
121+    await enablePushover(other);
122+    vi.restoreAllMocks();
123+    mockEndpoints({ [A]: 500, [B]: 200 }, pushes, true);
124+    expect(await runCycle(other)).toHaveLength(1);
125+    vi.restoreAllMocks();
126+    const retried: URLSearchParams[] = [];
127+    mockEndpoints({ [A]: 500, [B]: 200 }, retried);
128+    expect(await runCycle(other)).toEqual([]);
129+    expect(titles(retried)).toEqual([expect.stringMatching(/^DOWN: a\.test.* from apac$/)]);
130+  });
131+});
132+
133 describe("history", () => {
134   it("pages newest first and filters non-200", async () => {
135     const stub = await freshMonitor("weur");

 1@@ -10,6 +10,9 @@ export default defineConfig({
 2           ENDPOINTS: ["https://a.test/healthz", "https://b.test/healthz"],
 3           AUTH_USER: "admin",
 4           AUTH_PASS: "secret",
 5+          // Off unless a test enables it, whatever .dev.vars holds.
 6+          PUSHOVER_TOKEN: "",
 7+          PUSHOVER_USER: "",
 8         },
 9       },
10     }),