Justfile
1# Fleet workflow (README.md, docs/fleet-migration-plan.md). Runs from any
2# shell: the tools only the dev shell has (colmena, nvd, nixos-anywhere) are
3# called through `dev`; jq, ssh, nix and `secrets` come from the system.
4
5# The `secrets` store (age-encrypted, one file per secret).
6export SECRETS_DIR := env("SECRETS_DIR", home_directory() / "projects/data")
7
8# Runs a command from this flake's dev shell (the pinned colmena 0.5.0 etc.).
9dev := "nix develop " + quote(justfile_directory()) + " --command"
10
11# List recipes.
12default:
13 @just --list
14
15# Evaluate every host (fast sanity check; no builds).
16eval:
17 {{ dev }} colmena eval -E '{ nodes, ... }: builtins.mapAttrs (n: v: v.config.system.build.toplevel.drvPath) nodes'
18
19# Build host systems without deploying (all hosts, or one).
20build HOST="":
21 {{ dev }} colmena build {{ if HOST == "" { "" } else { "--on " + HOST } }}
22
23# Builds HOST's system, fetches the one it runs, and compares them with nvd.
24# "identical" means a deploy would change nothing.
25# Compare what the fleet would deploy to HOST with what HOST runs.
26diff-host HOST:
27 #!/usr/bin/env bash
28 set -euo pipefail
29 running="$(ssh "root@{{ HOST }}" readlink -f /run/current-system)"
30 new="$(nix build --no-link --print-out-paths '.#nixosConfigurations."{{ HOST }}".config.system.build.toplevel')"
31 echo "running: $running"
32 echo "fleet: $new"
33 if [ "$running" = "$new" ]; then
34 echo "identical: deploying {{ HOST }} changes nothing"
35 exit 0
36 fi
37 nix copy --no-check-sigs --from "ssh-ng://root@{{ HOST }}" "$running"
38 {{ dev }} nvd diff "$running" "$new"
39
40# Refuses to start if any secret HOST declares is missing from the store.
41# Extra args go to `colmena apply`: `just deploy HOST dry-activate`,
42# `just deploy HOST --reboot`.
43# Deploy HOST with Colmena (build, copy, upload keys, switch).
44deploy HOST *ARGS: (check-secrets HOST)
45 {{ dev }} colmena apply {{ ARGS }} --on {{ HOST }}
46
47# Every host with a role from PROJECT ("moods" or "sovrn") gets the new
48# version; the lock bump is the deploy log, so commit it afterwards.
49# Pick up PROJECT's last commit and deploy the hosts that run it.
50deploy-project PROJECT *ARGS:
51 #!/usr/bin/env bash
52 set -euo pipefail
53 nix flake update {{ PROJECT }}
54 hosts="$(jq -r --arg p "{{ PROJECT }}" '[to_entries[] | select(any(.value.roles[]?; . == $p or startswith($p + "-"))) | .key] | join(",")' hosts.json)"
55 [ -n "$hosts" ] || { echo "no host in hosts.json runs a {{ PROJECT }} role" >&2; exit 1; }
56 for h in ${hosts//,/ }; do just check-secrets "$h"; done
57 {{ dev }} colmena apply {{ ARGS }} --on "$hosts"
58 echo "deployed {{ PROJECT }} $(jq -r '.nodes["{{ PROJECT }}"].locked.rev' flake.lock) to $hosts; commit flake.lock"
59
60# Covers every project's secrets: reads the `secrets decrypt <path>` key
61# commands from each host's deployment.keys. With no HOST, checks all hosts.
62# Verify every secret a host declares decrypts from the store.
63check-secrets HOST="":
64 #!/usr/bin/env bash
65 set -euo pipefail
66 paths="$(nix eval --json .#colmenaHive.nodes --apply \
67 'ns: builtins.mapAttrs (n: v: map (k: builtins.elemAt k.keyCommand 2) (builtins.filter (k: k.keyCommand or null != null && builtins.length k.keyCommand == 3) (builtins.attrValues v.config.deployment.keys))) ns' |
68 jq -r --arg h "{{ HOST }}" 'to_entries[] | select($h == "" or .key == $h) | .value[]' | sort -u)"
69 missing=0
70 for p in $paths; do
71 # Byte count, not $(...): some keys are binary and may start with NUL.
72 if [ "$(secrets decrypt "$p" 2>/dev/null | head -c1 | wc -c)" = 1 ]; then :; else echo "missing: $p" >&2; missing=1; fi
73 done
74 [ "$missing" = 0 ] || exit 1
75 echo "secrets ok ({{ if HOST == "" { "all hosts" } else { HOST } }})"
76
77# SSH key authorized for root on fresh provider images (Hetzner's
78# HCLOUD_DEFAULT_SSH_KEYS); one of keys/admins.pub.
79SSH_KEY := env("FLEET_SSH_KEY", home_directory() / ".ssh/hetzner")
80
81# stateVersion recorded for newly installed standard hosts: the release they
82# were installed with. Never change it for an existing host.
83NEW_STATE_VERSION := "26.05"
84
85# Steps:
86# 0. netcup only: ask for a public key, check it is in keys/admins.pub,
87# ssh-copy-id it to root (password login; netcup images allow no key),
88# and check key login works. Its private key (the path without .pub) is
89# used for the rest instead of FLEET_SSH_KEY.
90# 1. Probe the provider image over SSH: architecture, install disk, IPv4
91# prefix and gateway, IPv6 address.
92# 2. Record the host in hosts.json (provider, system, disk, addresses,
93# stateVersion, roles), merged into an existing entry.
94# 3. `gen-host-secrets`: the SSH host key (servers/hosts/<fqdn>/, reused on
95# reinstalls) and the roles' host-scoped secrets.
96# 4. nixos-anywhere, installing the host key via --extra-files.
97# 5. Wait for the installed system, verify it presents the pinned key, then
98# pin it in ~/.ssh/known_hosts.
99#
100# ROLES is comma-separated (e.g. sovrn-cell, or moods,sovrn-metrics). Set
101# FLEET_YES=1 to skip the confirmation prompt.
102# Install NixOS on a fresh Hetzner or netcup box with nixos-anywhere. ERASES its disk.
103new-host IP HOSTNAME PROVIDER ROLES:
104 #!/usr/bin/env bash
105 set -euo pipefail
106 IP="{{ IP }}"
107 HOST="{{ HOSTNAME }}"
108 PROVIDER="{{ PROVIDER }}"
109 ROLES="{{ ROLES }}"
110 SSH_KEY="{{ SSH_KEY }}"
111 INVENTORY=hosts.json
112 fail() { echo "new-host: $*" >&2; exit 1; }
113
114 [[ "$IP" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]] || fail "IP must be an IPv4 address, got: $IP"
115 [[ "$HOST" =~ ^[a-z0-9-]+(\.[a-z0-9-]+)+$ ]] || fail "HOSTNAME must be a fully qualified lowercase name, got: $HOST"
116 case "$PROVIDER" in hetzner|netcup) ;; *) fail "PROVIDER must be hetzner or netcup, got: $PROVIDER" ;; esac
117 [[ "$ROLES" =~ ^[a-z0-9-]+(,[a-z0-9-]+)*$ ]] || fail "ROLES must be comma-separated role names, got: $ROLES"
118 for bin in secrets jq ssh ssh-keygen ssh-copy-id; do
119 command -v "$bin" >/dev/null || fail "$bin not on PATH"
120 done
121 if jq -e --arg h "$HOST" '.[$h].layout == "legacy"' "$INVENTORY" >/dev/null; then
122 fail "$HOST is a legacy-layout host; new-host installs the standard layout (see plan 2.8)"
123 fi
124 if jq -e --arg h "$HOST" 'has($h)' "$INVENTORY" >/dev/null; then
125 echo "note: $HOST is already in $INVENTORY; reinstalling (its entry will be updated)" >&2
126 fi
127 if [ "${FLEET_YES:-}" != 1 ]; then
128 echo "This installs NixOS ($ROLES) on $IP ($PROVIDER) as $HOST and ERASES the disk of $IP."
129 read -r -p "Type the hostname to continue: " answer
130 [ "$answer" = "$HOST" ] || fail "aborted"
131 fi
132
133 # 0. netcup images only allow root in with a password: authorize a key
134 # from keys/admins.pub, which then logs in to the NixOS host too. The
135 # image's host key is never trusted, as in step 1.
136 UNPINNED=(-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no)
137 if [ "$PROVIDER" = netcup ]; then
138 read -r -e -p "Public key to authorize for root@$IP (one of keys/admins.pub): " PUBKEY
139 PUBKEY="${PUBKEY/#\~/$HOME}"
140 [ -r "$PUBKEY" ] || fail "$PUBKEY not readable"
141 grep -qxF "$(cut -d' ' -f1,2 "$PUBKEY")" <(cut -d' ' -f1,2 keys/admins.pub) \
142 || fail "$PUBKEY is not in keys/admins.pub, so it couldn't log in once NixOS is installed"
143 SSH_KEY="${PUBKEY%.pub}"
144 [ -r "$SSH_KEY" ] || fail "private key $SSH_KEY (for $PUBKEY) not readable"
145 echo "Copying $PUBKEY to root@$IP; enter the root password from netcup when asked."
146 ssh-copy-id -i "$PUBKEY" "${UNPINNED[@]}" "root@$IP" || fail "ssh-copy-id to root@$IP failed"
147 ssh -i "$SSH_KEY" -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=10 -o LogLevel=ERROR \
148 "${UNPINNED[@]}" "root@$IP" true || fail "key login to root@$IP still fails after ssh-copy-id"
149 echo "Key login to root@$IP works."
150 fi
151 [ -r "$SSH_KEY" ] || fail "SSH key $SSH_KEY not readable (set FLEET_SSH_KEY)"
152
153 tmp="$(mktemp -d)"
154 trap 'rm -rf "$tmp"' EXIT
155
156 # 1. Probe. The provider image's host key is never trusted or recorded:
157 # it is about to be destroyed, and nixos-anywhere ignores known_hosts too.
158 SSH_OPTS=(-i "$SSH_KEY" -o IdentitiesOnly=yes -o ConnectTimeout=10 -o LogLevel=ERROR)
159 # One key=value per line, so a missing value can't shift the others.
160 probe="$(ssh "${SSH_OPTS[@]}" "${UNPINNED[@]}" "root@$IP" 'bash -s' <<'PROBE'
161 echo "arch=$(uname -m)"
162 echo "disk=$(lsblk -dpno NAME,TYPE | awk '$2 == "disk" && $1 ~ /^\/dev\/(sd|vd|nvme)/ { print $1; exit }')"
163 echo "ipv4=$(ip -4 -o addr show scope global | awk '{ print $4; exit }')"
164 echo "gateway4=$(ip -4 route show default | awk '{ print $3; exit }')"
165 echo "ipv6=$(ip -6 -o addr show scope global | awk '{ print $4; exit }')"
166 PROBE
167 )" || fail "cannot SSH to root@$IP with $SSH_KEY"
168 get() { sed -n "s/^$1=//p" <<<"$probe"; }
169 case "$(get arch)" in
170 x86_64) SYSTEM=x86_64-linux ;;
171 aarch64) SYSTEM=aarch64-linux ;;
172 *) fail "unsupported architecture: $(get arch)" ;;
173 esac
174 DISK="$(get disk)"
175 [ -n "$DISK" ] || fail "no install disk found on $IP"
176 IPV4_CIDR="$(get ipv4)"
177 [ "${IPV4_CIDR%/*}" = "$IP" ] || fail "$IP's image reports IPv4 ${IPV4_CIDR:-none}, not $IP"
178 IPV4_PREFIX="${IPV4_CIDR#*/}"
179 GATEWAY4="$(get gateway4)"
180 [ -n "$GATEWAY4" ] || fail "no IPv4 default gateway on $IP"
181 IPV6="$(get ipv6)"
182 echo "probe: $SYSTEM, disk $DISK, IPv4 $IPV4_CIDR via $GATEWAY4, IPv6 ${IPV6:-none}"
183 if [ -z "$IPV6" ]; then
184 echo "note: no IPv6 configured on the image; set ipv6 in $INVENTORY (e.g. \"2a01:...::1/64\") and deploy" >&2
185 fi
186
187 # 2. Inventory, merged into any existing entry so hand-set fields
188 # (settings, an existing stateVersion) survive a reinstall. The pinned key
189 # is filled in after step 3.
190 jq --arg h "$HOST" --arg provider "$PROVIDER" --arg system "$SYSTEM" --arg disk "$DISK" \
191 --arg ipv4 "$IP" --argjson prefix "$IPV4_PREFIX" --arg gateway4 "$GATEWAY4" --arg ipv6 "$IPV6" \
192 --arg sv "{{ NEW_STATE_VERSION }}" --arg roles "$ROLES" \
193 '.[$h] = ({stateVersion: $sv} + (.[$h] // {}) + {provider: $provider, system: $system, disk: $disk,
194 ipv4: $ipv4, ipv4Prefix: $prefix, ipv4Gateway: $gateway4,
195 ipv6: (if $ipv6 == "" then null else $ipv6 end), sshHostKey: null,
196 roles: ($roles | split(","))})' \
197 "$INVENTORY" > "$tmp/hosts.json"
198 mv "$tmp/hosts.json" "$INVENTORY"
199 jj st >/dev/null 2>&1 || true
200
201 # 3. Per-host secrets, including the SSH host key (reused on rebuilds).
202 just gen-host-secrets "$HOST"
203 mkdir -p "$tmp/extra/etc/ssh"
204 KEY="$tmp/extra/etc/ssh/ssh_host_ed25519_key"
205 (umask 077; secrets decrypt "servers/hosts/$HOST/ssh_host_ed25519_key" > "$KEY")
206 ssh-keygen -y -f "$KEY" > "$KEY.pub"
207 PUB="$(cut -d' ' -f1,2 "$KEY.pub")"
208 jq --arg h "$HOST" --arg key "$PUB" '.[$h].sshHostKey = $key' "$INVENTORY" > "$tmp/hosts.json"
209 mv "$tmp/hosts.json" "$INVENTORY"
210 jj st >/dev/null 2>&1 || true
211
212 # 4. Install (reboots into NixOS at the end).
213 {{ dev }} nixos-anywhere --flake ".#$HOST" -i "$SSH_KEY" --extra-files "$tmp/extra" --target-host "root@$IP"
214
215 # 5. Verify the pinned key, then trust it.
216 echo "$HOST,$IP $PUB" > "$tmp/known_hosts"
217 for _ in $(seq 1 60); do
218 if ssh "${SSH_OPTS[@]}" -o UserKnownHostsFile="$tmp/known_hosts" -o StrictHostKeyChecking=yes \
219 "root@$IP" true 2>/dev/null; then
220 just _pin-host-key "$HOST" "$IP" "$PUB"
221 ssh "${SSH_OPTS[@]}" "root@$IP" nixos-version
222 echo "$HOST is up on NixOS. Commit $INVENTORY, then deploy with: just deploy $HOST"
223 exit 0
224 fi
225 sleep 5
226 done
227 fail "$HOST did not come back with the pinned host key within 5 minutes"
228
229# Generates only what's missing; existing values are never rotated. The SSH
230# host key lives at servers/hosts/<fqdn>/ssh_host_ed25519_key; every other
231# host-scoped secret comes from the keys the host's roles declare
232# (<project>/hosts/<fqdn>/<name>), generated by that project's
233# gen-secret-<project> app. Values no generator makes (provider tokens) are
234# listed with the command to store them. DRY=1 shows what would happen.
235# Create HOST's missing per-host secrets in the store.
236gen-host-secrets HOST:
237 #!/usr/bin/env bash
238 set -euo pipefail
239 HOST="{{ HOST }}"
240 system="$(nix eval --raw --impure --expr builtins.currentSystem)"
241 tmp="$(mktemp -d)"
242 trap 'rm -rf "$tmp"' EXIT
243 paths="$(nix eval --json ".#nixosConfigurations.\"$HOST\".config.deployment.keys" --apply \
244 'ks: map (k: builtins.elemAt k.keyCommand 2) (builtins.filter (k: k.keyCommand or null != null && builtins.length k.keyCommand == 3) (builtins.attrValues ks))' |
245 jq -r --arg h "$HOST" '.[] | select(contains("/hosts/" + $h + "/"))' | sort -u)"
246 apps="$(nix eval --json ".#apps.$system" --apply builtins.attrNames | jq -r '.[]')"
247 manual=()
248 for secret in "servers/hosts/$HOST/ssh_host_ed25519_key" $paths; do
249 name="${secret##*/}"
250 project="${secret%%/*}"
251 if secrets decrypt "$secret" >/dev/null 2>&1; then
252 echo "exists $secret"
253 continue
254 fi
255 if [ "$name" = ssh_host_ed25519_key ]; then
256 gen=(sh -c 'ssh-keygen -q -t ed25519 -N "" -C "root@$1" -f "$2/key" && cat "$2/key"' _ "$HOST" "$tmp")
257 elif grep -qx "gen-secret-$project" <<<"$apps"; then
258 gen=(nix run ".#gen-secret-$project" -- "$name" "$HOST")
259 else
260 gen=(false)
261 fi
262 if [ "${DRY:-}" = 1 ]; then
263 # Run the generator but keep nothing: it says whether it can.
264 if "${gen[@]}" >/dev/null 2>&1; then echo "would gen $secret"; else echo "MISSING $secret (not generated)"; fi
265 elif "${gen[@]}" > "$tmp/value" 2>/dev/null; then
266 secrets encrypt "$secret" < "$tmp/value" 2>/dev/null
267 echo "generated $secret"
268 else
269 manual+=("$secret")
270 echo "MISSING $secret (not generated)"
271 fi
272 rm -f "$tmp/key" "$tmp/key.pub" "$tmp/value"
273 done
274 if [ ${#manual[@]} -gt 0 ]; then
275 echo
276 echo "Store these by hand (e.g. a provider API token):"
277 for secret in "${manual[@]}"; do echo " printf '%s' '<value>' | secrets encrypt $secret"; done
278 fi
279
280# Expected records come from hosts.json and sovrn's fleet.nix at the locked
281# input (what is deployed); see scripts/check-dns.sh. Needs dig and openssl.
282# Compare the sovrn.at zone (Marque record, nameservers, DNSSEC) with the fleet.
283check-dns: (_dns "check-dns")
284
285# Shows check-dns, the plan (records to remove/add in the managed rrsets),
286# asks, then writes the Marque record with goat (logged in as @rtw.run) and
287# re-checks once the nameservers have it. DRY=1 stops after the plan.
288# Converge the sovrn.at zone on the fleet through the Marque record.
289update-dns: (_dns "update-dns")
290
291_dns SCRIPT:
292 #!/usr/bin/env bash
293 set -euo pipefail
294 fleet="$(mktemp)"; trap 'rm -f "$fleet"' EXIT
295 nix eval --json --impure --expr \
296 'let f = import ((builtins.getFlake (toString ./.)).inputs.sovrn + "/nix/fleet.nix"); in { inherit (f) zone rootHost; }' \
297 2>/dev/null >"$fleet"
298 scripts/{{ SCRIPT }}.sh hosts.json "$fleet"
299
300# Use on a new workstation, or after a host was rebuilt elsewhere.
301# Pin every host in hosts.json into ~/.ssh/known_hosts.
302known-hosts:
303 #!/usr/bin/env bash
304 set -euo pipefail
305 jq -r 'to_entries[] | select(.value.sshHostKey != null) | "\(.key) \(.value.ipv4) \(.value.sshHostKey)"' hosts.json |
306 while read -r host ip keytype key; do
307 just _pin-host-key "$host" "$ip" "$keytype $key"
308 done
309
310# Replace the known_hosts entries for HOST and IP with KEY ("<type> <base64>").
311_pin-host-key HOST IP KEY:
312 #!/usr/bin/env bash
313 set -euo pipefail
314 KH="$HOME/.ssh/known_hosts"
315 touch "$KH"
316 ssh-keygen -R "{{ HOST }}" -f "$KH" >/dev/null 2>&1 || true
317 ssh-keygen -R "{{ IP }}" -f "$KH" >/dev/null 2>&1 || true
318 echo "{{ HOST }},{{ IP }} {{ KEY }}" >> "$KH"
319 rm -f "$KH.old"
320 echo "known_hosts: pinned {{ HOST }} ({{ IP }})"
321
322# Run after `just update-nixpkgs` in sovrn and moods. Fails unless both lock
323# the same revision; the fleet then pins it (flake and devenv) and re-locks
324# both projects.
325# Move the fleet's nixpkgs to the revision sovrn and moods lock.
326sync-nixpkgs:
327 #!/usr/bin/env bash
328 set -euo pipefail
329 sovrn="$(jq -r .nodes.nixpkgs.locked.rev ~/projects/sovrn/flake.lock)"
330 moods="$(jq -r .nodes.nixpkgs.locked.rev ~/projects/moods/flake.lock)"
331 [ "$sovrn" = "$moods" ] || { echo "sovrn locks $sovrn, moods locks $moods: bump them to the same revision first" >&2; exit 1; }
332 sed -i "s|nixpkgs\\.url = \"github:NixOS/nixpkgs/[0-9a-f]*\";|nixpkgs.url = \"github:NixOS/nixpkgs/$sovrn\";|" flake.nix
333 sed -i "s|url: github:NixOS/nixpkgs/[0-9a-f]*$|url: github:NixOS/nixpkgs/$sovrn|" devenv.yaml
334 nix flake update nixpkgs sovrn moods
335 devenv update nixpkgs
336 just eval >/dev/null
337 echo "nixpkgs -> $sovrn (fleet, devenv, sovrn, moods); commit flake.nix, flake.lock, devenv.yaml and devenv.lock"
