servers

 1@@ -280,14 +280,22 @@ gen-host-secrets HOST:
 2 # Expected records come from hosts.json and sovrn's fleet.nix at the locked
 3 # input (what is deployed); see scripts/check-dns.sh. Needs dig and openssl.
 4 # Compare the sovrn.at zone (Marque record, nameservers, DNSSEC) with the fleet.
 5-check-dns:
 6+check-dns: (_dns "check-dns")
 7+
 8+# Shows check-dns, the plan (records to remove/add in the managed rrsets),
 9+# asks, then writes the Marque record with goat (logged in as @rtw.run) and
10+# re-checks once the nameservers have it. DRY=1 stops after the plan.
11+# Converge the sovrn.at zone on the fleet through the Marque record.
12+update-dns: (_dns "update-dns")
13+
14+_dns SCRIPT:
15     #!/usr/bin/env bash
16     set -euo pipefail
17     fleet="$(mktemp)"; trap 'rm -f "$fleet"' EXIT
18     nix eval --json --impure --expr \
19       'let f = import ((builtins.getFlake (toString ./.)).inputs.sovrn + "/nix/fleet.nix"); in { inherit (f) mailDomain rootHost; }' \
20       2>/dev/null >"$fleet"
21-    scripts/check-dns.sh hosts.json "$fleet"
22+    scripts/{{ SCRIPT }}.sh hosts.json "$fleet"
23 
24 # Use on a new workstation, or after a host was rebuilt elsewhere.
25 # Pin every host in hosts.json into ~/.ssh/known_hosts.

1@@ -33,6 +33,7 @@ The recipes run from any shell: the tools only the dev shell provides (colmena,
2 | `just deploy-project <sovrn\|moods>` | Pick up the project's last commit (`nix flake update <p>`) and deploy every host running one of its roles. Commit `flake.lock` afterwards: it is the deploy log. |
3 | `just check-secrets [fqdn]` | Verify every secret a host declares decrypts from the store. |
4 | `just check-dns` | Compare the sovrn.at zone with the fleet: the Marque record in the PDS, each authoritative nameserver, and DNSSEC validation of the TLSA records. Expected records come from `hosts.json` and sovrn's `fleet.nix` (see `scripts/check-dns.sh`). Needs `dig` and `openssl`. |
5+| `just update-dns` | Converge the sovrn.at zone on the fleet: shows `check-dns`, then the plan (records to remove and add, only in the rrsets the fleet manages), asks, and writes the Marque record with goat (logged in as @rtw.run; `swapRecord` guards against concurrent edits), then re-checks once the nameservers have it. `DRY=1` stops after the plan. |
6 | `just known-hosts` | Pin every host's SSH key (from `hosts.json`) in `~/.ssh/known_hosts`. |
7 | `just sync-nixpkgs` | After `just update-nixpkgs` in sovrn and moods: move the fleet to the revision they lock. |
8 | `just new-host <ipv4> <fqdn> <hetzner\|netcup> <roles>` | Install NixOS on a fresh box with nixos-anywhere (**erases it**): probes the image, records it in `hosts.json`, generates its secrets and SSH host key, installs, pins the key. |

 1@@ -33,6 +33,10 @@
 2 #
 3 # Usage: check-dns.sh <hosts.json> <fleet.json>
 4 #   fleet.json: {"mailDomain": ..., "rootHost": ...}
 5+# With CHECK_DNS_STATE=<dir>, also leaves what it compared there for
 6+# update-dns.sh: expected.tsv, absent.tsv, record.json (the getRecord
 7+# response, with its CID) and marque.tsv (one normalised line per entry of
 8+# record.json's records, in the same order).
 9 # Needs curl, jq, dig, openssl on PATH.
10 set -euo pipefail
11 
12@@ -90,7 +94,7 @@ ok() { echo "  ${green}✓${reset} $*"; }
13 # --- Expected --------------------------------------------------------------
14 
15 expected="$(mktemp)"; absent="$(mktemp)"; marque="$(mktemp)"
16-trap 'rm -f "$expected" "$absent" "$marque"' EXIT
17+trap 'rm -f "$expected" "$absent" "$marque" "$marque.ordered"' EXIT
18 
19 addrs() { # host -> A/AAAA lines for name $2
20   jq -r --arg h "$1" --arg n "$2" '.[$h] |
21@@ -139,7 +143,14 @@ jq -r --arg z "$zone" '.value.records[] |
22   [ (if .name == "@" then $z else "\(.name).\($z)" end),
23     .recordType,
24     (if .recordType == "MX" then "\(.priority) \(.value)" else .value end) ] | @tsv' <<<"$record" |
25-  norm | sort -u >"$marque"
26+  norm >"$marque.ordered"
27+sort -u "$marque.ordered" >"$marque"
28+if [ -n "${CHECK_DNS_STATE:-}" ]; then
29+  cp "$expected" "$CHECK_DNS_STATE/expected.tsv"
30+  cp "$absent" "$CHECK_DNS_STATE/absent.tsv"
31+  cp "$marque.ordered" "$CHECK_DNS_STATE/marque.tsv"
32+  printf '%s\n' "$record" >"$CHECK_DNS_STATE/record.json"
33+fi
34 echo "  cid $(jq -r .cid <<<"$record"), $(wc -l <"$marque") records, written $(jq -r .value.createdAt <<<"$record")"
35 
36 while IFS=$'\t' read -r name type; do

  1@@ -0,0 +1,147 @@
  2+#!/usr/bin/env bash
  3+# Converges the sovrn mail zone on what the fleet expects (`just update-dns`):
  4+#
  5+#   1. Runs check-dns.sh and shows its report.
  6+#   2. Plans against the Marque record that report read: in every rrset the
  7+#      fleet manages (check-dns.sh's expected and absent sets), records that
  8+#      shouldn't be there are removed and missing ones added. Records at
  9+#      unmanaged names are left exactly as they are.
 10+#   3. Shows the plan and asks for confirmation (FLEET_YES=1 skips it;
 11+#      DRY=1 stops after the plan).
 12+#   4. Checks goat's session is the zone record's owner: the session DID is
 13+#      $MARQUE_REPO and $MARQUE_HANDLE resolves to it.
 14+#   5. Writes the merged record with putRecord through goat, with swapRecord
 15+#      set to the CID the plan was made from: if anything (the Marque web UI,
 16+#      Marque's own normalisation) wrote the record since, the PDS refuses
 17+#      and nothing changes; run it again.
 18+#   6. Waits for every nameserver to serve a new SOA serial and runs
 19+#      check-dns.sh again.
 20+#
 21+# Marque serves the zone from this one record, so a write replaces the whole
 22+# zone; the plan is the only change. New records get TTL 300 like the rest.
 23+#
 24+# Usage: update-dns.sh <hosts.json> <fleet.json>
 25+# Needs check-dns.sh's tools plus goat (logged in: `goat account login`).
 26+set -euo pipefail
 27+
 28+here="$(dirname "$0")"
 29+INVENTORY="$1"
 30+FLEET="$2"
 31+MARQUE_REPO="${MARQUE_REPO:-did:plc:zpqnooj2vuoju6ssulbpdwxp}"
 32+MARQUE_HANDLE="${MARQUE_HANDLE:-rtw.run}"
 33+TTL=300
 34+export MARQUE_REPO
 35+
 36+command -v goat >/dev/null || { echo "update-dns: goat not on PATH (the devenv shell has it)" >&2; exit 2; }
 37+
 38+if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then
 39+  red=$'\e[31m' green=$'\e[32m' bold=$'\e[1m' reset=$'\e[0m'
 40+else
 41+  red="" green="" bold="" reset=""
 42+fi
 43+
 44+state="$(mktemp -d)"
 45+trap 'rm -rf "$state"' EXIT
 46+
 47+# 1. Report.
 48+echo "${bold}== check-dns${reset}"
 49+if CHECK_DNS_STATE="$state" "$here/check-dns.sh" "$INVENTORY" "$FLEET"; then
 50+  exit 0
 51+fi
 52+[ -s "$state/record.json" ] || { echo "update-dns: check-dns didn't get as far as the Marque record" >&2; exit 2; }
 53+
 54+zone="$(jq -r .mailDomain "$FLEET")"
 55+cid="$(jq -r .cid "$state/record.json")"
 56+count="$(jq '.value.records | length' "$state/record.json")"
 57+# marque.tsv must line up with the record's entries (a tab or newline inside
 58+# a value would shift it).
 59+[ "$(wc -l <"$state/marque.tsv")" = "$count" ] ||
 60+  { echo "update-dns: couldn't line up the record's $count entries; not touching it" >&2; exit 2; }
 61+
 62+# 2. Plan. drop: 0-based indices into .records; add: expected lines missing
 63+# from the record.
 64+awk -F'\t' '
 65+  FILENAME ~ /expected/ { want[$0]; managed[$1 FS $2]; next }
 66+  FILENAME ~ /absent/   { managed[$1 FS $2]; next }
 67+  {
 68+    i = FNR - 1
 69+    if (($1 FS $2) in managed && (!($0 in want) || ($0 in seen))) print i
 70+    seen[$0]
 71+  }' "$state/expected.tsv" "$state/absent.tsv" "$state/marque.tsv" >"$state/drop"
 72+awk -F'\t' 'NR == FNR { have[$0]; next } !($0 in have)' \
 73+  "$state/marque.tsv" "$state/expected.tsv" >"$state/add"
 74+
 75+echo
 76+echo "${bold}== plan for $zone (record $cid)${reset}"
 77+if [ ! -s "$state/drop" ] && [ ! -s "$state/add" ]; then
 78+  echo "  nothing the Marque record can fix: the discrepancies above are elsewhere"
 79+  echo "  (nameservers still propagating, DNSSEC, or a certificate that couldn't be fetched)"
 80+  exit 1
 81+fi
 82+while read -r i; do
 83+  line="$(sed -n "$((i + 1))p" "$state/marque.tsv")"
 84+  note=""; ! grep -qxF "$line" "$state/expected.tsv" || note=" (duplicate)"
 85+  awk -F'\t' -v r="$red" -v z="$reset" -v n="$note" '{ printf "  %s- remove %s %s %s%s%s\n", r, $1, $2, $3, n, z }' <<<"$line"
 86+done <"$state/drop"
 87+awk -F'\t' -v g="$green" -v z="$reset" '{ printf "  %s+ add    %s %s %s (ttl '"$TTL"')%s\n", g, $1, $2, $3, z }' "$state/add"
 88+
 89+# The merged record: unmanaged entries untouched, in their order.
 90+jq --arg z "$zone" --argjson ttl "$TTL" \
 91+  --slurpfile drop <(jq -s . "$state/drop") \
 92+  --rawfile add "$state/add" '
 93+  def rel: if . == $z then "@" else rtrimstr("." + $z) end;
 94+  .value
 95+  | .records = ([.records | to_entries[] | select(.key as $k | ($drop[0] | index($k)) == null) | .value]
 96+      + [$add | split("\n")[] | select(. != "") | split("\t") as [$n, $t, $v]
 97+         | if $t == "MX" then ($v | split(" ")) as [$p, $h]
 98+             | {name: ($n | rel), recordType: $t, value: $h, priority: ($p | tonumber), ttl: $ttl}
 99+           else {name: ($n | rel), recordType: $t, value: $v, ttl: $ttl} end])
100+  | .createdAt = (now | todate)' "$state/record.json" >"$state/new.json"
101+
102+before="$count"
103+after="$(jq '.records | length' "$state/new.json")"
104+echo "  $before records -> $after"
105+# Never write a zone that lost more than the plan says (or everything).
106+[ "$after" -gt 0 ] && [ "$after" = "$((before - $(wc -l <"$state/drop") + $(wc -l <"$state/add")))" ] ||
107+  { echo "update-dns: the merged record doesn't add up; not writing it" >&2; exit 2; }
108+
109+[ "${DRY:-}" != 1 ] || { echo "update-dns: DRY=1, nothing written"; exit 1; }
110+
111+# 3. Confirm.
112+echo
113+if [ "${FLEET_YES:-}" != 1 ]; then
114+  read -r -p "Write this to at://$MARQUE_REPO/at.marque.dns/$zone? Type yes: " answer
115+  [ "$answer" = yes ] || { echo "update-dns: nothing written"; exit 1; }
116+fi
117+
118+# 4. Whose session is this?
119+session_did="$(goat account check-auth 2>/dev/null | awk '/^DID:/ { print $2 }')"
120+[ "$session_did" = "$MARQUE_REPO" ] ||
121+  { echo "update-dns: goat is logged in as '${session_did:-nobody}', not $MARQUE_REPO; run: goat account login -u $MARQUE_HANDLE" >&2; exit 2; }
122+handle_did="$(goat resolve "$MARQUE_HANDLE" 2>/dev/null | jq -r .id)"
123+[ "$handle_did" = "$MARQUE_REPO" ] ||
124+  { echo "update-dns: $MARQUE_HANDLE resolves to '${handle_did:-nothing}', not $MARQUE_REPO" >&2; exit 2; }
125+echo "goat: logged in as $MARQUE_HANDLE ($session_did)"
126+
127+serials() {
128+  for ns in $(dig +short NS "$zone"); do dig +short +norec SOA "$zone" @"$ns" | awk '{ print $3 }'; done | sort -u | paste -sd' '
129+}
130+old_serials="$(serials)"
131+
132+# 5. Write (swapRecord: only over the record the plan was made from).
133+jq -n --arg repo "$MARQUE_REPO" --arg rkey "$zone" --arg swap "$cid" --slurpfile v "$state/new.json" \
134+  '{repo: $repo, collection: "at.marque.dns", rkey: $rkey, validate: false, swapRecord: $swap, record: $v[0]}' |
135+  goat xrpc procedure @pds com.atproto.repo.putRecord - 'Content-Type:application/json' >"$state/put.json" ||
136+  { echo "${red}update-dns: putRecord failed (if InvalidSwap: the record changed since the plan; run again)${reset}" >&2; cat "$state/put.json" >&2; exit 1; }
137+echo "written: cid $(jq -r .cid "$state/put.json")"
138+
139+# 6. Wait for the nameservers (Marque took ~45 s in testing), then re-check.
140+echo "waiting for every nameserver to leave serial $old_serials ..."
141+for _ in $(seq 1 40); do
142+  now="$(serials)"
143+  if [ "$now" != "$old_serials" ] && [ "$(wc -w <<<"$now")" = 1 ]; then break; fi
144+  sleep 15
145+done
146+echo
147+echo "${bold}== check-dns${reset}"
148+"$here/check-dns.sh" "$INVENTORY" "$FLEET"