hive.nix
1# The Colmena hive, built from the inventory in hosts.json. Each entry:
2#
3# "<fqdn>": {
4# "roles": ["moods", "sovrn-metrics", ...], (see `roleModules`)
5# "system": "x86_64-linux" | "aarch64-linux",
6# "nixpkgs": "shared" | "stable", (default "shared")
7# "layout": "standard" | "legacy", (default "standard")
8# "sshHostKey": "ssh-ed25519 AAAA...", (pinned; `just known-hosts`)
9# "ipv4": "<public IPv4>",
10# "ipv6": "<addr>/64" | null,
11#
12# standard layout only (disko + modules/vm.nix + a provider module):
13# "stateVersion": "26.05", (required; never change it)
14# "provider": "hetzner" | "netcup",
15# "disk": "/dev/sda" | "/dev/vda",
16# "ipv4Prefix": 22, "ipv4Gateway": "<IPv4>", (netcup)
17# "settings": { "<project>": { ... } } (per-host project overrides)
18# }
19#
20# Legacy hosts were installed before the fleet (nixos-infect): they import
21# hosts/<fqdn with dashes>/ for hardware, networking and stateVersion
22# instead of the standard layout.
23#
24# The inventory is JSON so recipes can edit it with jq. Flakes only see files
25# the VCS tracks: a new file must be tracked (jj st) before evaluation sees it.
26{ inputs }:
27
28let
29 inherit (inputs.nixpkgs) lib;
30
31 hosts = lib.mapAttrs (name: h: h // { inherit name; }) (
32 builtins.fromJSON (builtins.readFile ./hosts.json)
33 );
34
35 channels = {
36 shared = inputs.nixpkgs;
37 stable = inputs.nixpkgs-stable;
38 };
39 channelOf =
40 host:
41 channels.${host.nixpkgs or "shared"}
42 or (throw "${host.name}: unknown nixpkgs \"${host.nixpkgs}\" in hosts.json (known: ${lib.concatStringsSep ", " (lib.attrNames channels)})");
43
44 # sovrn and moods build their dev shells and VM tests on the nixpkgs they
45 # lock. Hosts run their modules on the fleet's, so the two must be the same
46 # revision or a host would run builds nobody tested.
47 nixpkgsMatches = lib.all (
48 project:
49 let
50 theirs = inputs.${project}.inputs.nixpkgs.rev;
51 in
52 lib.assertMsg (theirs == inputs.nixpkgs.rev) ''
53 ${project} locks nixpkgs ${theirs}, the fleet pins ${inputs.nixpkgs.rev}.
54 Bump them together: `just update-nixpkgs` in ${project}, then
55 `just sync-nixpkgs` here.''
56 ) [ "sovrn" "moods" ];
57
58 # Role name in hosts.json -> NixOS modules. Project roles come from the
59 # projects' flakes; per-host project settings from hosts.json "settings".
60 roleModules = host: {
61 forge = [
62 ./roles/forge
63 { _module.args.pgit = inputs.pgit; }
64 ];
65 moods = [
66 inputs.moods.nixosModules.app
67 { moods.settings = (host.settings or { }).moods or { }; }
68 ];
69 sovrn-metrics = [
70 inputs.sovrn.nixosModules.metrics
71 {
72 sovrn.fleet = (host.settings or { }).sovrn or { };
73 # The DANE watch: every cell's TLSA must match its certificate; the
74 # relays must have none (the backup MX fails open, bug 209274a).
75 sovrn.daneCheck = {
76 hosts = map (h: h.name) (hostsWithRole "sovrn-cell");
77 failOpenHosts = map (h: h.settings.sovrnRelay.hostname) (hostsWithRole "sovrn-relay");
78 };
79 }
80 ];
81 sovrn-cell = [
82 inputs.sovrn.nixosModules.cell
83 (
84 { config, ... }:
85 let
86 cells = map (h: h.name) (hostsWithRole "sovrn-cell");
87 inherit (config.sovrn.fleet) rootHost;
88 in
89 {
90 # Exactly one cell serves sovrn.at (the landing page and its
91 # certificate): sovrn's fleet.rootHost, one value for every cell.
92 assertions = [
93 {
94 assertion = !(((host.settings or { }).sovrn or { }) ? rootHost);
95 message = "${host.name}: settings.sovrn.rootHost in hosts.json would give cells different landing cells; change rootHost in sovrn's nix/fleet.nix instead";
96 }
97 {
98 assertion = lib.elem rootHost cells;
99 message = "sovrn's rootHost ${rootHost} is not a sovrn-cell in hosts.json (cells: ${lib.concatStringsSep ", " cells}): sovrn.at would be served nowhere";
100 }
101 ];
102 }
103 )
104 {
105 sovrn.fleet = (host.settings or { }).sovrn or { };
106 sovrn.cell.pdsAutoProvision = host.pdsAutoProvision or false;
107 # Only the inventory's relays may reach the relay ingress port.
108 sovrn.cell.relaySources = lib.concatMap hostAddresses (hostsWithRole "sovrn-relay");
109 }
110 ];
111 sovrn-relay = [
112 inputs.sovrn.nixosModules.relay
113 {
114 sovrn.fleet = (host.settings or { }).sovrn or { };
115 # hostname (required) and any other sovrn.relay.* from
116 # hosts.json settings.sovrnRelay; the cells it covers are every
117 # sovrn-cell host in the inventory.
118 sovrn.relay = ((host.settings or { }).sovrnRelay or { }) // {
119 cells = map (h: h.name) (hostsWithRole "sovrn-cell");
120 };
121 }
122 ];
123 };
124
125 # sovrn's telemetry edge comes with its metrics and relay roles, which
126 # label their own services by their hostnames (cell and host). On a host
127 # they share with other projects, `cell` says whose the rest is: moods'
128 # (units moods*), else the fleet's; `host` stays the machine.
129 sharedTelemetry =
130 host:
131 let
132 has = role: lib.elem role (host.roles or [ ]);
133 in
134 lib.optional (has "sovrn-metrics" || has "sovrn-relay") {
135 sovrn.telemetry.edge = {
136 cell = "servers";
137 units = lib.optionalAttrs (has "moods") { moods.cell = "moods"; };
138 };
139 };
140
141 hostsWithRole = role: lib.filter (h: lib.elem role (h.roles or [ ])) (lib.attrValues hosts);
142 # A host's public addresses as firewall sources (IPv6 without its prefix
143 # length: the host's own address, not its /64).
144 hostAddresses =
145 h:
146 lib.optional (h ? ipv4) h.ipv4
147 ++ lib.optional ((h.ipv6 or null) != null) (builtins.head (lib.splitString "/" h.ipv6));
148 modulesForRole =
149 host: role:
150 (roleModules host).${role}
151 or (throw "${host.name}: unknown role \"${role}\" in hosts.json (known: ${lib.concatStringsSep ", " (lib.attrNames (roleModules host))})");
152
153 providers = {
154 hetzner = ./modules/hetzner.nix;
155 netcup = ./modules/netcup.nix;
156 };
157 provider =
158 host:
159 let
160 name = host.provider or (throw "${host.name}: standard hosts need \"provider\" in hosts.json");
161 in
162 providers.${name}
163 or (throw "${host.name}: unknown provider \"${name}\" in hosts.json (known: ${lib.concatStringsSep ", " (lib.attrNames providers)})");
164
165 layouts = {
166 # hosts/<fqdn with dashes>/, if present, holds settings for that host only.
167 standard =
168 host:
169 [
170 inputs.disko.nixosModules.disko
171 ./modules/base.nix
172 ./modules/vm.nix
173 (provider host)
174 {
175 system.stateVersion =
176 host.stateVersion or (throw "${host.name}: standard hosts need \"stateVersion\" in hosts.json");
177 }
178 ]
179 ++ lib.optional (builtins.pathExists ./hosts/${dirName host}) ./hosts/${dirName host};
180 # What `nixos-rebuild --flake .#<name>` built before the fleet (minus
181 # agenix since plan 2.4); hosts/<name>/ brings its own hardware,
182 # networking and stateVersion.
183 legacy = host: [ ./hosts/${dirName host} ];
184 };
185 layout =
186 host:
187 let
188 name = host.layout or "standard";
189 in
190 (layouts.${name} or (throw "${host.name}: unknown layout \"${name}\" in hosts.json")) host;
191
192 dirName = host: lib.replaceStrings [ "." ] [ "-" ] host.name;
193
194 modulesFor =
195 host:
196 let
197 nixpkgs = channelOf host;
198 in
199 layout host
200 ++ [
201 ./modules/fleet.nix
202 ./modules/caddy.nix
203 ./modules/shell.nix
204 { fleet.roles = host.roles or [ ]; }
205 ]
206 ++ lib.concatMap (modulesForRole host) (host.roles or [ ])
207 ++ sharedTelemetry host
208 ++ [
209 {
210 # Colmena evaluates nixpkgs outside nixosSystem, which would label the
211 # system "pre-git"; record the locked revision instead (as sovrn's and
212 # moods' hives do).
213 system.nixos.versionSuffix = ".${builtins.substring 0 8 (nixpkgs.lastModifiedDate or "0")}.${nixpkgs.shortRev or "dirty"}";
214 system.nixos.revision = nixpkgs.rev or null;
215 }
216 # Legacy hosts were built by nixosSystem, which also sets the flake
217 # source (nix registry, NIX_PATH). Standard hosts come from Colmena
218 # hives that never did; each keeps what it had.
219 (lib.optionalAttrs ((host.layout or "standard") == "legacy") {
220 nixpkgs.flake.source = nixpkgs.outPath;
221 })
222 ];
223in
224assert nixpkgsMatches;
225{
226 meta = {
227 nixpkgs = import inputs.nixpkgs { system = "x86_64-linux"; };
228 nodeNixpkgs = lib.mapAttrs (_: host: import (channelOf host) { inherit (host) system; }) hosts;
229 nodeSpecialArgs = lib.mapAttrs (
230 _: host:
231 {
232 inherit host;
233 }
234 // lib.optionalAttrs ((host.layout or "standard") == "legacy") {
235 # What the pre-fleet flake passed to hosts/<name>/.
236 flakeRoot = ./.;
237 flakeHostName = dirName host;
238 }
239 ) hosts;
240 };
241}
242// lib.mapAttrs (_: host: {
243 imports = modulesFor host;
244 deployment = {
245 # Connect by name, so the deployer's ~/.ssh/config applies. Needs DNS for
246 # the host; known_hosts pins both name and IP (`just known-hosts`).
247 targetHost = host.name;
248 targetUser = "root";
249 # Build here; aarch64 through binfmt emulation, as in moods.
250 buildOnTarget = false;
251 tags = host.roles or [ ];
252 };
253}) hosts
