servers

 1@@ -4,6 +4,7 @@ Personal git hosting on infra.rtw.run: [soft-serve](https://github.com/charmbrac
 2 
 3 | Name | Serves |
 4 | --- | --- |
 5+| `infra.rtw.run` | `/healthz` only: the forge's health for the monitor. Points at the host. |
 6 | `git.kilimanjaro.io` | soft-serve's HTTP (clone, go-get) through Caddy. Points at the host. `/` is a 404 by design. |
 7 | `kilimanjaro.io` | `/var/www/code` through Caddy. Proxied by Cloudflare. |
 8 | `www.kilimanjaro.io` | Redirect to kilimanjaro.io. Proxied by Cloudflare. |
 9@@ -19,11 +20,12 @@ Personal git hosting on infra.rtw.run: [soft-serve](https://github.com/charmbrac
10 
11 | File | What |
12 | --- | --- |
13-| `default.nix` | soft-serve, the hook, the backup job and timer, Caddy's three sites. |
14+| `default.nix` | soft-serve, the hook, the backup and health jobs and their timers, Caddy's four sites. |
15 | `soft-serve/config.yaml` | `/etc/soft-serve/config.yaml`. soft-serve restarts when it changes. |
16 | `soft-serve/hooks/post-receive` | Global hook, linked from `/var/soft/data/hooks`. Backgrounds `forge-rebuild-site` and returns at once. |
17 | `rebuild-site.sh` | `forge-rebuild-site`: rebuilds every public, non-hidden repo's pages and the index. |
18 | `backup.sh` | Daily (20:30 UTC) `rclone sync` of `/var/soft/data` to `r2:soft-serve`, with a Pushover notification. |
19+| `healthz.sh` | `forge-healthz`, every 5 minutes: disk, load, backup age, soft-serve. Writes the `/healthz` response (below). |
20 | `recover.sh` | Restores `/var/soft/data` from R2 and rebuilds the site. |
21 
22 ## On the host
23@@ -31,11 +33,33 @@ Personal git hosting on infra.rtw.run: [soft-serve](https://github.com/charmbrac
24 | Path | What |
25 | --- | --- |
26 | `/var/soft/data` | soft-serve: repos, `soft-serve.db`, its SSH host keys (`ssh/`). Backed up. |
27+| `/run/forge-healthz` | The `/healthz` response: `ok.json` (Caddy answers 200) or `degraded.json` (503). Neither until the first run after boot (404). |
28+| `/var/lib/forge-healthz/load` | 24 hours of load samples, for `max_24h`. |
29+| `/var/lib/forge-backup/last-success` | Touched by each successful backup; `/healthz` fails when it's over 26 hours old. |
30 | `/var/www/code` | The generated site. Not backed up: `forge-rebuild-site` recreates it. |
31 | `/var/log/pgit.log` | Output of every site build. |
32 | `/var/lib/caddy` | Caddy's certificates and ACME account (Caddy runs as `caddy`). |
33 | `/var/lib/servers-keys` | `pushover.env`, `rclone.conf` (Colmena keys). |
34 
35+## /healthz
36+
37+`https://infra.rtw.run/healthz` answers 200 when every check passes, 503 otherwise, with the body sovrn's `metrics-healthz` uses:
38+
39+```json
40+{
41+  "status": "ok",
42+  "checked_at": "2026-10-11T05:15:18Z",
43+  "checks": {
44+    "disk": { "status": "ok", "detail": "used=18% threshold=80%" },
45+    "load": { "status": "ok", "detail": "load=0.00 0.00 0.00 max_24h=0.14 threshold=1.9 (15m)" },
46+    "backup": { "status": "ok", "detail": "last_success=2026-10-10T20:31:00Z age=8h max_age=26h" },
47+    "soft-serve": { "status": "ok", "detail": "active" }
48+  }
49+}
50+```
51+
52+A failing check also has `error`. Caddy serves the file `forge-healthz` last wrote, so the answer can be up to 5 minutes old (`checked_at`). If the script itself fails, it replaces `ok.json` with a `degraded.json` naming the journal.
53+
54 ## Things that bit before
55 
56 - **The hook must not hold the push open.** soft-serve 0.11 passes git, and so the hook, an extra pipe (fd 5) and ends the push only at EOF on it. The background job closes every descriptor above 2 first; otherwise each push waits for the whole site build.

1@@ -31,6 +31,8 @@ log "starting soft-serve backup"
2 
3 if $RCLONE sync -L /var/soft/data/ r2:soft-serve/; then
4     log "completed soft-serve backup"
5+    # forge-healthz reports the backup stale when this is over 26h old.
6+    touch /var/lib/forge-backup/last-success
7     send_notification "success"
8 else
9     log "soft-serve backup failed"

  1@@ -1,6 +1,7 @@
  2 # Role: forge (infra.rtw.run). Personal git hosting: soft-serve behind Caddy,
  3 # and kilimanjaro.io, the static site pgit builds from the public repos.
  4 #
  5+#   infra.rtw.run         /healthz only (forge-healthz)
  6 #   git.kilimanjaro.io    soft-serve's HTTP (git smart HTTP, go-get), :23232
  7 #   kilimanjaro.io        /var/www/code, written by the post-receive hook
  8 #   www.kilimanjaro.io    redirect to kilimanjaro.io
  9@@ -14,6 +15,8 @@
 10 #                       dir; rebuilds /var/www/code in the background
 11 #   backup              daily `rclone sync` of /var/soft/data to r2:soft-serve
 12 #                       (recover.sh restores it)
 13+#   forge-healthz       every 5 minutes: disk, load, backup age, soft-serve;
 14+#                       writes the /healthz response Caddy serves
 15 #
 16 # Secrets (modules/secrets.nix): pushover.env (shared) for notifications,
 17 # rclone.conf (per host) for R2.
 18@@ -41,6 +44,16 @@ let
 19     text = builtins.readFile ./rebuild-site.sh;
 20   };
 21   softServeConfig = ./soft-serve/config.yaml;
 22+  healthz = pkgs.writeShellApplication {
 23+    name = "forge-healthz";
 24+    runtimeInputs = with pkgs; [
 25+      coreutils
 26+      gawk
 27+      jq
 28+      systemd
 29+    ];
 30+    text = builtins.readFile ./healthz.sh;
 31+  };
 32 
 33   securityHeaders = ''
 34     header {
 35@@ -140,6 +153,8 @@ in
 36       ExecStart = "${backupScript}/bin/backup";
 37       User = "root";
 38       EnvironmentFile = pushoverKey.path;
 39+      # last-success, for forge-healthz
 40+      StateDirectory = "forge-backup";
 41     };
 42     environment = {
 43       RCLONE = "${pkgs.rclone}/bin/rclone";
 44@@ -158,6 +173,28 @@ in
 45     };
 46   };
 47 
 48+  # The /healthz response, as a file for Caddy (see healthz.sh). Its output
 49+  # stays in /run after each run; the load samples persist in /var/lib.
 50+  systemd.services.forge-healthz = {
 51+    description = "Write the forge's /healthz response";
 52+    serviceConfig = {
 53+      Type = "oneshot";
 54+      ExecStart = lib.getExe healthz;
 55+      RuntimeDirectory = "forge-healthz";
 56+      RuntimeDirectoryPreserve = true;
 57+      StateDirectory = "forge-healthz";
 58+    };
 59+  };
 60+
 61+  systemd.timers.forge-healthz = {
 62+    description = "Write the forge's /healthz response (every 5 minutes)";
 63+    wantedBy = [ "timers.target" ];
 64+    timerConfig = {
 65+      OnBootSec = "1min";
 66+      OnUnitActiveSec = "5min";
 67+    };
 68+  };
 69+
 70   # Interactive `rclone` as root (e.g. recover.sh) finds the same config. The
 71   # file itself stays root-only.
 72   environment.variables.RCLONE_CONFIG = rcloneKey.path;
 73@@ -171,6 +208,33 @@ in
 74     # INFO (renewals show in the journal) and no access logs. Keep both.
 75     logFormat = lib.mkForce "level INFO";
 76 
 77+    # ok.json -> 200, otherwise degraded.json -> 503, or 404 when neither
 78+    # exists yet. Never cached: the monitor must see each run's answer.
 79+    virtualHosts."infra.rtw.run" = {
 80+      logFormat = null;
 81+      extraConfig = ''
 82+        ${securityHeaders}
 83+        handle /healthz {
 84+            root * /run/forge-healthz
 85+            header Cache-Control "no-store"
 86+            @ok file /ok.json
 87+            handle @ok {
 88+                rewrite * /ok.json
 89+                file_server
 90+            }
 91+            handle {
 92+                rewrite * /degraded.json
 93+                file_server {
 94+                    status 503
 95+                }
 96+            }
 97+        }
 98+        handle {
 99+            respond 404
100+        }
101+      '';
102+    };
103+
104     virtualHosts."git.kilimanjaro.io" = {
105       logFormat = null;
106       extraConfig = ''

 1@@ -0,0 +1,94 @@
 2+# forge-healthz: the forge's health for GET https://infra.rtw.run/healthz,
 3+# polled by the monitor (~/projects/monitor). Runs every 5 minutes and writes
 4+# the response for Caddy to serve as a static file:
 5+#
 6+#   /run/forge-healthz/ok.json        every check passes   -> 200
 7+#   /run/forge-healthz/degraded.json  any check fails      -> 503
 8+#
 9+# At most one of them exists; with neither (just booted) Caddy answers 404.
10+# The body has the shape of sovrn's metrics-healthz:
11+# {"status": "ok"|"degraded", "checked_at", "checks": {<name>: {"status":
12+# "ok"|"fail", "detail", "error"}}}.
13+#
14+# Checks (thresholds from the Pushover-era monitor.sh):
15+#   disk        root filesystem % used
16+#   load        15-minute load average; the 24h maximum of the 1-minute
17+#               samples is information only
18+#   backup      age of the last successful backup (backup.sh stamps it)
19+#   soft-serve  the unit is active
20+
21+OUT_DIR=/run/forge-healthz
22+STATE_DIR=/var/lib/forge-healthz
23+BACKUP_STAMP=/var/lib/forge-backup/last-success
24+
25+DISK_THRESHOLD=80              # % used on /
26+LOAD_THRESHOLD=1.9             # 2 cores at ~95%
27+BACKUP_MAX_AGE=$((26 * 3600))  # daily at 20:30 UTC, plus time to run
28+
29+# publish NAME BODY: make NAME.json the one Caddy serves.
30+publish() {
31+  local other=ok
32+  [[ $1 == ok ]] && other=degraded
33+  printf '%s\n' "$2" >"$OUT_DIR/.$1.json.tmp"
34+  mv "$OUT_DIR/.$1.json.tmp" "$OUT_DIR/$1.json"
35+  rm -f "$OUT_DIR/$other.json"
36+}
37+
38+# If this script dies, the old answer must not stand: say so, as a 503.
39+published=
40+on_exit() {
41+  [[ -n $published ]] && return
42+  publish degraded "{\"status\":\"degraded\",\"checked_at\":\"$(date -u +%FT%TZ)\",\"checks\":{\"healthz\":{\"status\":\"fail\",\"error\":\"forge-healthz failed; see journalctl -u forge-healthz\"}}}"
43+}
44+trap on_exit EXIT
45+
46+now=$(date +%s)
47+checks='{}'
48+
49+# check NAME ok|fail DETAIL
50+check() {
51+  checks=$(jq -c --arg name "$1" --arg status "$2" --arg detail "$3" \
52+    '.[$name] = {status: $status, detail: $detail}
53+     + (if $status == "ok" then {} else {error: "\($name): \($detail)"} end)' \
54+    <<<"$checks")
55+}
56+
57+# exceeds A B: true when A > B (decimals).
58+exceeds() {
59+  awk -v a="$1" -v b="$2" 'BEGIN { exit !(a > b) }'
60+}
61+
62+# disk
63+used=$(df --output=pcent / | tail -n 1 | tr -dc 0-9)
64+detail="used=${used}% threshold=${DISK_THRESHOLD}%"
65+if ((used > DISK_THRESHOLD)); then check disk fail "$detail"; else check disk ok "$detail"; fi
66+
67+# load: keep 24 hours of 1-minute samples, one per run.
68+read -r load1 load5 load15 _ </proc/loadavg
69+echo "$now $load1" >>"$STATE_DIR/load"
70+awk -v cutoff=$((now - 86400)) '$1 >= cutoff' "$STATE_DIR/load" >"$STATE_DIR/load.tmp"
71+mv "$STATE_DIR/load.tmp" "$STATE_DIR/load"
72+max24h=$(awk 'NR == 1 || $2 > max { max = $2 } END { print max }' "$STATE_DIR/load")
73+detail="load=${load1} ${load5} ${load15} max_24h=${max24h} threshold=${LOAD_THRESHOLD} (15m)"
74+if exceeds "$load15" "$LOAD_THRESHOLD"; then check load fail "$detail"; else check load ok "$detail"; fi
75+
76+# backup
77+if [[ -f $BACKUP_STAMP ]]; then
78+  last=$(stat -c %Y "$BACKUP_STAMP")
79+  age=$((now - last))
80+  detail="last_success=$(date -u -d "@$last" +%FT%TZ) age=$((age / 3600))h max_age=$((BACKUP_MAX_AGE / 3600))h"
81+  if ((age > BACKUP_MAX_AGE)); then check backup fail "$detail"; else check backup ok "$detail"; fi
82+else
83+  check backup fail "no successful backup recorded ($BACKUP_STAMP)"
84+fi
85+
86+# soft-serve
87+state=$(systemctl is-active soft-serve || true)
88+if [[ $state == active ]]; then check soft-serve ok "$state"; else check soft-serve fail "$state"; fi
89+
90+status=$(jq -r 'if all(.[]; .status == "ok") then "ok" else "degraded" end' <<<"$checks")
91+body=$(jq -n --arg status "$status" --arg at "$(date -u -d "@$now" +%FT%TZ)" --argjson checks "$checks" \
92+  '{status: $status, checked_at: $at, checks: $checks}')
93+publish "$status" "$body"
94+published=1
95+echo "$status: $(jq -c . <<<"$checks")"