servers

 1@@ -277,6 +277,18 @@ gen-host-secrets HOST:
 2       for secret in "${manual[@]}"; do echo "  printf '%s' '<value>' | secrets encrypt $secret"; done
 3     fi
 4 
 5+# Expected records come from hosts.json and sovrn's fleet.nix at the locked
 6+# input (what is deployed); see scripts/check-dns.sh. Needs dig and openssl.
 7+# Compare the sovrn.at zone (Marque record, nameservers, DNSSEC) with the fleet.
 8+check-dns:
 9+    #!/usr/bin/env bash
10+    set -euo pipefail
11+    fleet="$(mktemp)"; trap 'rm -f "$fleet"' EXIT
12+    nix eval --json --impure --expr \
13+      'let f = import ((builtins.getFlake (toString ./.)).inputs.sovrn + "/nix/fleet.nix"); in { inherit (f) mailDomain rootHost; }' \
14+      2>/dev/null >"$fleet"
15+    scripts/check-dns.sh hosts.json "$fleet"
16+
17 # Use on a new workstation, or after a host was rebuilt elsewhere.
18 # Pin every host in hosts.json into ~/.ssh/known_hosts.
19 known-hosts:

1@@ -32,6 +32,7 @@ The recipes run from any shell: the tools only the dev shell provides (colmena,
2 | `just deploy <fqdn> [args]` | Check its secrets, then `colmena apply`. `just deploy <fqdn> --reboot` makes the new system the boot default and reboots into it. |
3 | `just deploy-project <sovrn\|moods>` | Pick up the project's last commit (`nix flake update <p>`) and deploy every host running one of its roles. Commit `flake.lock` afterwards: it is the deploy log. |
4 | `just check-secrets [fqdn]` | Verify every secret a host declares decrypts from the store. |
5+| `just check-dns` | Compare the sovrn.at zone with the fleet: the Marque record in the PDS, each authoritative nameserver, and DNSSEC validation of the TLSA records. Expected records come from `hosts.json` and sovrn's `fleet.nix` (see `scripts/check-dns.sh`). Needs `dig` and `openssl`. |
6 | `just known-hosts` | Pin every host's SSH key (from `hosts.json`) in `~/.ssh/known_hosts`. |
7 | `just sync-nixpkgs` | After `just update-nixpkgs` in sovrn and moods: move the fleet to the revision they lock. |
8 | `just new-host <ipv4> <fqdn> <hetzner\|netcup> <roles>` | Install NixOS on a fresh box with nixos-anywhere (**erases it**): probes the image, records it in `hosts.json`, generates its secrets and SSH host key, installs, pins the key. |

  1@@ -0,0 +1,209 @@
  2+#!/usr/bin/env bash
  3+# Compares the sovrn mail zone with what the fleet says it should be, in one
  4+# report (`just check-dns`). Exits 1 on any discrepancy.
  5+#
  6+# Expected records, all derived (nothing is listed by hand):
  7+#   cells (hosts.json, role sovrn-cell)
  8+#     <cell>                    A / AAAA       the host's ipv4 / ipv6
  9+#     *<pdsOriginSuffix>        CNAME <cell>   the cell's PDS wildcard
 10+#     _25._tcp.<cell>           TLSA 3 1 1     SHA-256 of the SPKI the cell
 11+#                                              serves on :25 right now
 12+#   relay (hosts.json, role sovrn-relay, settings.sovrnRelay.hostname)
 13+#     <relay>                   A / AAAA       the host's ipv4 / ipv6
 14+#     _25._tcp.<relay>          no TLSA        the backup MX fails open
 15+#                                              (bug 209274a): it must be a
 16+#                                              validated NXDOMAIN, not a record
 17+#   apex (sovrn's fleet.nix at the locked input: mailDomain, rootHost)
 18+#     <mailDomain>              A / AAAA       rootHost's addresses
 19+#     <mailDomain>              MX 10 rootHost
 20+#   SMTP2GO branded names (bug e76f226)
 21+#     dkim.<mailDomain>         CNAME dkim.smtp2go.net
 22+#     return.<mailDomain>       CNAME return.smtp2go.net
 23+#
 24+# Checked against:
 25+#   marque   the zone record in the PDS (at.marque.dns/<mailDomain>), the
 26+#            source of truth Marque serves from. Records at names nobody
 27+#            manages are listed for information, not flagged.
 28+#   NS       every authoritative nameserver, queried directly for each
 29+#            managed name (catches Marque lag or ingest problems), plus their
 30+#            SOA serials.
 31+#   DNSSEC   a validating resolver ($RESOLVER) must return the cells' TLSA
 32+#            and the relay's TLSA denial with the AD flag: DANE senders defer
 33+#            on anything that doesn't validate.
 34+#
 35+# Usage: check-dns.sh <hosts.json> <fleet.json>
 36+#   fleet.json: {"mailDomain": ..., "rootHost": ...}
 37+# Needs curl, jq, dig, openssl on PATH.
 38+set -euo pipefail
 39+
 40+INVENTORY="$1"
 41+FLEET="$2"
 42+# The PDS repo holding the at.marque.dns record (@rtw.run).
 43+MARQUE_REPO="${MARQUE_REPO:-did:plc:zpqnooj2vuoju6ssulbpdwxp}"
 44+RESOLVER="${RESOLVER:-1.1.1.1}"
 45+
 46+for t in curl jq dig openssl; do
 47+  command -v "$t" >/dev/null || { echo "check-dns: $t not on PATH" >&2; exit 2; }
 48+done
 49+
 50+zone="$(jq -r .mailDomain "$FLEET")"
 51+root_host="$(jq -r .rootHost "$FLEET")"
 52+
 53+# Records are tab-separated lines: name, type, value. Names are FQDNs without
 54+# the trailing dot, lowercase; values are normalised the same way dig prints
 55+# them (TLSA hex lowercase, unsplit; MX "<prio> <host>").
 56+norm() {
 57+  awk -F'\t' -v OFS='\t' '{
 58+    n = tolower($1); sub(/\.$/, "", n)
 59+    t = toupper($2); v = $3
 60+    if (t == "CNAME" || t == "NS") { v = tolower(v); sub(/\.$/, "", v) }
 61+    else if (t == "MX") { split(v, a, " "); h = tolower(a[2]); sub(/\.$/, "", h); v = a[1] " " h }
 62+    else if (t == "TLSA") {
 63+      split(v, a, " "); hex = ""
 64+      for (i = 4; i in a; i++) hex = hex a[i]
 65+      v = a[1] " " a[2] " " a[3] " " tolower(hex)
 66+    }
 67+    else if (t == "AAAA") v = tolower(v)
 68+    print n, t, v
 69+  }'
 70+}
 71+
 72+spki_sha256() {
 73+  echo | timeout 20 openssl s_client -connect "$1:25" -starttls smtp -servername "$1" 2>/dev/null |
 74+    openssl x509 -noout -pubkey 2>/dev/null |
 75+    openssl pkey -pubin -outform DER 2>/dev/null |
 76+    sha256sum | cut -d' ' -f1
 77+}
 78+EMPTY_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
 79+
 80+problems=0
 81+problem() { echo "  ✗ $*"; problems=$((problems + 1)); }
 82+ok() { echo "  ✓ $*"; }
 83+
 84+# --- Expected --------------------------------------------------------------
 85+
 86+expected="$(mktemp)"; absent="$(mktemp)"; marque="$(mktemp)"
 87+trap 'rm -f "$expected" "$absent" "$marque"' EXIT
 88+
 89+addrs() { # host -> A/AAAA lines for name $2
 90+  jq -r --arg h "$1" --arg n "$2" '.[$h] |
 91+    (if .ipv4 then [$n, "A", .ipv4] else empty end),
 92+    (if .ipv6 then [$n, "AAAA", (.ipv6 | split("/")[0])] else empty end) | @tsv' "$INVENTORY"
 93+}
 94+
 95+{
 96+  for cell in $(jq -r 'to_entries[] | select(.value.roles | index("sovrn-cell")) | .key' "$INVENTORY"); do
 97+    addrs "$cell" "$cell"
 98+    suffix="$(jq -r --arg h "$cell" '.[$h].pdsOriginSuffix // empty' "$INVENTORY")"
 99+    [ -z "$suffix" ] || printf '*%s\tCNAME\t%s\n' "$suffix" "$cell"
100+    hash="$(spki_sha256 "$cell")"
101+    if [ "$hash" = "$EMPTY_SHA256" ]; then
102+      echo "check-dns: couldn't fetch $cell's certificate on :25; its TLSA isn't checked" >&2
103+      problems=$((problems + 1))
104+    else
105+      printf '_25._tcp.%s\tTLSA\t3 1 1 %s\n' "$cell" "$hash"
106+    fi
107+  done
108+  jq -r 'to_entries[] | select(.value.roles | index("sovrn-relay")) | [.key, .value.settings.sovrnRelay.hostname] | @tsv' "$INVENTORY" |
109+    while IFS=$'\t' read -r host relay; do addrs "$host" "$relay"; done
110+  addrs "$root_host" "$zone"
111+  printf '%s\tMX\t10 %s\n' "$zone" "$root_host"
112+  printf 'dkim.%s\tCNAME\tdkim.smtp2go.net\n' "$zone"
113+  printf 'return.%s\tCNAME\treturn.smtp2go.net\n' "$zone"
114+} >"$expected"
115+# Normalised outside the group: it runs in this shell, so problem counts stick.
116+norm <"$expected" | sort -u >"$expected.n" && mv "$expected.n" "$expected"
117+
118+# (name, type) pairs that must have no record.
119+jq -r 'to_entries[] | select(.value.roles | index("sovrn-relay")) | "_25._tcp.\(.value.settings.sovrnRelay.hostname)\tTLSA"' "$INVENTORY" |
120+  awk -F'\t' -v OFS='\t' '{ print tolower($1), $2 }' | sort -u >"$absent"
121+
122+# Managed rrsets: every (name, type) the fleet has an opinion on.
123+managed() { { cut -f1,2 "$expected"; cat "$absent"; } | sort -u; }
124+values() { awk -F'\t' -v n="$2" -v t="$3" '$1 == n && $2 == t { print $3 }' "$1" | sort; }
125+
126+# --- Marque (the PDS record) ----------------------------------------------
127+
128+echo "marque: at://$MARQUE_REPO/at.marque.dns/$zone"
129+pds="$(curl -sf "https://plc.directory/$MARQUE_REPO" | jq -r '.service[] | select(.id == "#atproto_pds") | .serviceEndpoint')"
130+record="$(curl -sf "$pds/xrpc/com.atproto.repo.getRecord?repo=$MARQUE_REPO&collection=at.marque.dns&rkey=$zone")" ||
131+  { echo "check-dns: couldn't fetch the Marque record from $pds" >&2; exit 2; }
132+jq -r --arg z "$zone" '.value.records[] |
133+  [ (if .name == "@" then $z else "\(.name).\($z)" end),
134+    .recordType,
135+    (if .recordType == "MX" then "\(.priority) \(.value)" else .value end) ] | @tsv' <<<"$record" |
136+  norm | sort -u >"$marque"
137+echo "  cid $(jq -r .cid <<<"$record"), $(wc -l <"$marque") records, written $(jq -r .value.createdAt <<<"$record")"
138+
139+while IFS=$'\t' read -r name type; do
140+  want="$(values "$expected" "$name" "$type" | paste -sd' ')"
141+  have="$(values "$marque" "$name" "$type" | paste -sd' ')"
142+  if [ "$want" = "$have" ]; then
143+    ok "$name $type ${want:-(none)}"
144+  else
145+    problem "$name $type: want [$want] have [$have]"
146+  fi
147+done < <(managed)
148+
149+echo "  unmanaged (information only):"
150+awk -F'\t' 'NR == FNR { m[$1 FS $2]; next }
151+  !(($1 FS $2) in m) { v = $3; if (length(v) > 60) v = substr(v, 1, 57) "..."; printf "    %s %s %s\n", $1, $2, v }' \
152+  <(managed) "$marque"
153+
154+# --- Authoritative nameservers --------------------------------------------
155+
156+mapfile -t nameservers < <(dig +short NS "$zone" | sed 's/\.$//' | sort)
157+echo
158+echo "nameservers: ${nameservers[*]}"
159+serials=""
160+for ns in "${nameservers[@]}"; do
161+  serials+="$(dig +short +norec SOA "$zone" @"$ns" | awk '{ print $3 }') "
162+done
163+if [ "$(tr ' ' '\n' <<<"$serials" | sed '/^$/d' | sort -u | wc -l)" = 1 ]; then
164+  ok "SOA serial ${serials%% *} on all"
165+else
166+  problem "SOA serials differ: $serials(${nameservers[*]}): a change is still propagating"
167+fi
168+
169+while IFS=$'\t' read -r name type; do
170+  want="$(values "$expected" "$name" "$type" | paste -sd' ')"
171+  bad=""
172+  for ns in "${nameservers[@]}"; do
173+    have="$(dig +short +norec "$type" "$name" @"$ns" | sed "s/^/$name\t$type\t/" | norm | cut -f3 | sort | paste -sd' ')"
174+    [ "$want" = "$have" ] || bad+=" $ns=[$have]"
175+  done
176+  if [ -z "$bad" ]; then
177+    ok "$name $type"
178+  else
179+    problem "$name $type: want [$want]$bad"
180+  fi
181+done < <(managed)
182+
183+# --- DNSSEC (DANE) ---------------------------------------------------------
184+
185+echo
186+echo "dnssec via $RESOLVER:"
187+while IFS=$'\t' read -r name type; do
188+  out="$(dig +dnssec "$type" "$name" @"$RESOLVER")"
189+  status="$(grep -o 'status: [A-Z]*' <<<"$out" | cut -d' ' -f2)"
190+  ad="$(grep -c '^;; flags:.* ad' <<<"$out" || true)"
191+  if grep -qx "$name"$'\t'"$type" "$absent"; then
192+    if [ "$status" = NXDOMAIN ] || { [ "$status" = NOERROR ] && ! grep -q $'\tIN\tTLSA\t' <<<"$out"; }; then
193+      [ "$ad" = 1 ] && ok "$name $type: validated denial ($status)" ||
194+        problem "$name $type: $status without AD; DANE senders can't prove it absent and defer"
195+    else
196+      problem "$name $type: expected no record, got $status"
197+    fi
198+  else
199+    [ "$status" = NOERROR ] && [ "$ad" = 1 ] && ok "$name $type: validated" ||
200+      problem "$name $type: status $status, AD=$ad"
201+  fi
202+done < <(managed | awk -F'\t' '$2 == "TLSA"')
203+
204+echo
205+if [ "$problems" = 0 ]; then
206+  echo "check-dns: $zone matches the fleet"
207+else
208+  echo "check-dns: $problems discrepancies in $zone"
209+  exit 1
210+fi