ci

 1@@ -483,11 +483,24 @@ Isolation (protects the host's config and services, not against attack)
 2 | Concern | Control |
 3 |---|---|
 4 | Writes to `/etc`, units, users, system profile | `ProtectSystem=strict`, non-root `ci` user, not in `trusted-users`. Only an explicit deploy action can change host config |
 5-| Port clashes (devenv postgres vs. production postgres) | Network namespace per segment via `pasta`, with egress everywhere |
 6+| Port clashes (devenv postgres vs. production postgres) | Opt-in network namespace per segment via `pasta` (egress, `pastaHostPorts` forwarded to the host). **Off by default**: in pasta's user namespace commands run as uid 0, root-owned files look unowned (OpenSSH refuses its config, so no SSH and no deploys) and the hostname reads `pasta-<host>`. A systemd-made namespace with pasta attached from outside would avoid that, but pasta's own sandboxing refuses to open the unit's namespace (tried; see bug 7a3d5cf/db62e70 notes) |
 7 | CPU / memory / IO starvation | `ci.slice` with low `CPUWeight`/`IOWeight`, `MemoryMax`; swap or zram so builds can't OOM soft-serve; 1–2 concurrent runs |
 8 | Disk | Run dirs expire after N days; host `nix.gc` and `min-free`/`max-free` |
 9 | Nix store | Shared through the host nix-daemon socket; append-only, so growth is disk usage, not pollution |
10 
11+The runner daemon runs as root (it creates transient units for `ci`;
12+granting `ci` polkit's `manage-units` would let a segment start root units),
13+hands each run directory and its repo's HOME to `ci` before a segment
14+starts, and is reachable on localhost only. Segments get a fixed PATH
15+(engine, bash, coreutils, git, openssh, nix, jq, curl, just, devenv), `NIX_REMOTE=daemon`,
16+`CI_SELF_HOST` (deploy recipes put that host last) and credentials: the age
17+identity (`SECRETS_IDENTITY`) and an SSH key, found by the `ci` user's
18+static `~/.ssh/config` (`IdentityFile ${CREDENTIALS_DIRECTORY}/ssh-key`; OpenSSH reads the passwd home,
19+and `${VAR}` expands per unit), with strict host keys except the forge on
20+localhost. `nix/module.nix` packages all of this as
21+`services.ci.{bridge,runner,appview}`; `nix/tests/e2e.nix` checks it in a
22+VM.
23+
24 Multi-arch: jobs use the host nix-daemon, which routes derivations by
25 `system`: `boot.binfmt.emulatedSystems = [ "aarch64-linux" ]` (as the fleet
26 already does with `buildOnTarget = false`), and optionally `nix.buildMachines`

 1@@ -7,6 +7,7 @@ import (
 2 	"log/slog"
 3 	"os"
 4 	"path/filepath"
 5+	"strconv"
 6 	"strings"
 7 
 8 	"git.kilimanjaro.io/ci/internal/eventlog"
 9@@ -17,6 +18,17 @@ func init() {
10 	commands["runner"] = command{"runner daemon: spindle-compatible host for the engine (serve)", cmdRunner}
11 }
12 
13+func credentials(identity, sshKey string) map[string]string {
14+	c := map[string]string{}
15+	if identity != "" {
16+		c["age-identity"] = identity
17+	}
18+	if sshKey != "" {
19+		c["ssh-key"] = sshKey
20+	}
21+	return c
22+}
23+
24 type listFlag []string
25 
26 func (l *listFlag) String() string     { return strings.Join(*l, ",") }
27@@ -42,7 +54,9 @@ func cmdRunner(args []string) int {
28 	slice := fs.String("slice", "ci.slice", "systemd slice for segments")
29 	user := fs.String("user", "ci", "user segments run as")
30 	identity := fs.String("identity", "", "age identity loaded into each segment (LoadCredential, SECRETS_IDENTITY)")
31+	sshKey := fs.String("ssh-key", "", "SSH private key loaded into each segment (git, colmena, ssh to the fleet)")
32 	pasta := fs.String("pasta", "", "pasta binary: give each segment its own network namespace (egress only)")
33+	pastaPorts := fs.String("pasta-host-ports", "", "with --pasta: TCP ports of the segment's localhost forwarded to the host's, comma-separated (e.g. 23231 for the forge)")
34 	var props, env listFlag
35 	fs.Var(&props, "property", "extra unit property, e.g. MemoryMax=8G (repeatable)")
36 	fs.Var(&env, "setenv", "extra environment for segments, NAME=value (repeatable)")
37@@ -84,11 +98,20 @@ func cmdRunner(args []string) int {
38 		cfg.Notify = runner.Ntfy(*ntfy, *tokens["ntfy"], *appview, *ntfySuccess, logger)
39 	}
40 	segEnv := append([]string{"CI_CLONE_URL=" + *cloneURL}, env...)
41+	var hostPorts []int
42+	for _, p := range strings.FieldsFunc(*pastaPorts, func(r rune) bool { return r == ',' }) {
43+		n, err := strconv.Atoi(strings.TrimSpace(p))
44+		if err != nil {
45+			fmt.Fprintf(os.Stderr, "ci runner serve: --pasta-host-ports: %q is not a port\n", p)
46+			return exitUsage
47+		}
48+		hostPorts = append(hostPorts, n)
49+	}
50 	var l runner.Launcher
51 	switch *launcher {
52 	case "systemd":
53 		l = &runner.Systemd{Engine: exe, Shell: *shell, LocksDir: cfg.LocksDir, Slice: *slice, User: *user,
54-			CacheDir: cfg.CacheDir, Identity: *identity, Pasta: *pasta, Properties: props, Env: segEnv}
55+			CacheDir: cfg.CacheDir, Credentials: credentials(*identity, *sshKey), Pasta: *pasta, PastaHostPorts: hostPorts, Properties: props, Env: segEnv}
56 	case "local":
57 		l = &runner.Local{Engine: exe, Shell: *shell, LocksDir: cfg.LocksDir, Env: segEnv}
58 	default:

 1@@ -39,12 +39,9 @@ func cmdRunnerExec(args []string) int {
 2 		fmt.Fprintln(os.Stderr, "ci:", err)
 3 		return exitUsage
 4 	}
 5-	// systemd LoadCredential= puts the runner's age identity here; the
 6-	// secrets tool reads SECRETS_IDENTITY.
 7-	if cd := os.Getenv("CREDENTIALS_DIRECTORY"); cd != "" && os.Getenv("SECRETS_IDENTITY") == "" {
 8-		if p := filepath.Join(cd, "age-identity"); fileExists(p) {
 9-			_ = os.Setenv("SECRETS_IDENTITY", p)
10-		}
11+	if err := useCredentials(os.Getenv("CREDENTIALS_DIRECTORY")); err != nil {
12+		fmt.Fprintln(os.Stderr, "ci:", err)
13+		return exitFailure
14 	}
15 	dir := engine.Dir(fs.Arg(0))
16 	e, err := engine.Open(dir, ex, engine.Options{Detach: true, LocksDir: *locks})
17@@ -74,6 +71,24 @@ func cmdRunnerExec(args []string) int {
18 	return exitCode(st.Status)
19 }
20 
21+// useCredentials wires up what systemd's LoadCredential= put in dir: the age
22+// identity becomes SECRETS_IDENTITY (the secrets tool). The SSH key is found
23+// by the ci user's static ~/.ssh/config (nix/module.nix), whose IdentityFile
24+// is ${CREDENTIALS_DIRECTORY}/ssh-key; here we only make sure the segment's
25+// $HOME/.ssh exists for its known_hosts.local.
26+func useCredentials(dir string) error {
27+	if dir == "" {
28+		return nil
29+	}
30+	if p := filepath.Join(dir, "age-identity"); fileExists(p) && os.Getenv("SECRETS_IDENTITY") == "" {
31+		_ = os.Setenv("SECRETS_IDENTITY", p)
32+	}
33+	if home := os.Getenv("HOME"); home != "" && fileExists(filepath.Join(dir, "ssh-key")) {
34+		return os.MkdirAll(filepath.Join(home, ".ssh"), 0o700)
35+	}
36+	return nil
37+}
38+
39 func fileExists(p string) bool {
40 	fi, err := os.Stat(p)
41 	return err == nil && fi.Mode().IsRegular()

 1@@ -50,6 +50,9 @@
 2         default = ci;
 3       });
 4 
 5+      # services.ci.{bridge,runner,appview}; see nix/module.nix.
 6+      nixosModules.default = import ./nix/module.nix self;
 7+
 8       overlays.default = final: _prev: {
 9         ci = self.packages.${final.stdenv.hostPlatform.system}.ci;
10       };
11@@ -71,6 +74,10 @@
12             runHook postCheck
13           '';
14         });
15+      }
16+      // lib.optionalAttrs pkgs.stdenv.hostPlatform.isx86_64 {
17+        # a NixOS VM: soft-serve + bridge + runner (systemd) + appview
18+        e2e = import ./nix/tests/e2e.nix { inherit self pkgs; };
19       });
20     };
21 }

  1@@ -3,9 +3,13 @@ package runner
  2 import (
  3 	"context"
  4 	"fmt"
  5+	"io/fs"
  6+	"maps"
  7 	"os"
  8 	"os/exec"
  9+	"os/user"
 10 	"path/filepath"
 11+	"slices"
 12 	"strconv"
 13 	"strings"
 14 	"syscall"
 15@@ -124,11 +128,16 @@ type Systemd struct {
 16 	User     string // default ci
 17 	// CacheDir holds per-repo HOMEs (shared state between runs of a repo).
 18 	CacheDir string
 19-	// Identity, if set, is loaded as the credential age-identity.
 20-	Identity string
 21+	// Credentials are loaded into each segment with LoadCredential=
 22+	// (name -> file, read by systemd as root): age-identity becomes
 23+	// SECRETS_IDENTITY, ssh-key the segment's SSH identity (runner-exec).
 24+	Credentials map[string]string
 25 	// Pasta, if set, is the pasta binary: each segment gets its own network
 26 	// namespace with egress, so dev services can't clash with the host's.
 27 	Pasta string
 28+	// PastaHostPorts are TCP ports on the segment's localhost forwarded to
 29+	// the host's (e.g. the forge's SSH port, so a localhost clone URL works).
 30+	PastaHostPorts []int
 31 	// Properties are extra unit properties (e.g. MemoryMax=8G).
 32 	Properties []string
 33 	// Env is extra environment, e.g. CI_CLONE_URL.
 34@@ -160,8 +169,8 @@ func (s *Systemd) Command(j Job) []string {
 35 		"-p", "StandardOutput=append:" + filepath.Join(j.Dir, fmt.Sprintf("segment-%d.log", j.Segment)),
 36 		"-p", "StandardError=inherit",
 37 	}
 38-	if s.Identity != "" {
 39-		argv = append(argv, "-p", "LoadCredential=age-identity:"+s.Identity)
 40+	for _, name := range slices.Sorted(maps.Keys(s.Credentials)) {
 41+		argv = append(argv, "-p", "LoadCredential="+name+":"+s.Credentials[name])
 42 	}
 43 	for _, e := range s.Env {
 44 		argv = append(argv, "--setenv="+e)
 45@@ -171,8 +180,14 @@ func (s *Systemd) Command(j Job) []string {
 46 	}
 47 	argv = append(argv, "--")
 48 	if s.Pasta != "" {
 49-		// egress through user-mode NAT; no ports forwarded in either direction
 50-		argv = append(argv, s.Pasta, "--config-net", "--quiet", "-t", "none", "-u", "none", "-T", "none", "-U", "none", "--")
 51+		// pasta spawns the engine in new user and network namespaces:
 52+		// egress through user-mode NAT, nothing forwarded in, the listed
 53+		// ports of its localhost forwarded to the host's. Caveats: the
 54+		// engine runs as uid 0 of that user namespace (tools that refuse
 55+		// root, like postgres' initdb, won't run) and the hostname reads
 56+		// pasta-<host>. Opt-in; see DESIGN.md §10.
 57+		argv = append(argv, s.PastaCommand()...)
 58+		argv = append(argv, "--")
 59 	}
 60 	argv = append(argv, s.Engine, "runner-exec", "--shell", orDefault(s.Shell, "devenv"), "--locks-dir", s.LocksDir)
 61 	argv = append(argv, j.Args...)
 62@@ -180,9 +195,16 @@ func (s *Systemd) Command(j Job) []string {
 63 }
 64 
 65 func (s *Systemd) Start(ctx context.Context, j Job) (string, error) {
 66-	if err := os.MkdirAll(filepath.Join(s.CacheDir, "home", j.Repo), 0o755); err != nil {
 67+	home := filepath.Join(s.CacheDir, "home", j.Repo)
 68+	if err := os.MkdirAll(home, 0o755); err != nil {
 69+		return "", err
 70+	}
 71+	// the runner prepares as root; the segment runs as User and writes the
 72+	// run dir, its repo's HOME and the locks
 73+	if err := chownTree(orDefault(s.User, "ci"), j.Dir, home, s.LocksDir); err != nil {
 74 		return "", err
 75 	}
 76+
 77 	argv := s.Command(j)
 78 	out, err := exec.CommandContext(ctx, argv[0], argv[1:]...).CombinedOutput()
 79 	if err != nil {
 80@@ -191,6 +213,19 @@ func (s *Systemd) Start(ctx context.Context, j Job) (string, error) {
 81 	return "unit:" + UnitName(j.Run, j.Segment), nil
 82 }
 83 
 84+// PastaCommand is pasta's part of a segment's command line.
 85+func (s *Systemd) PastaCommand() []string {
 86+	hostPorts := "none"
 87+	if len(s.PastaHostPorts) > 0 {
 88+		var ps []string
 89+		for _, p := range s.PastaHostPorts {
 90+			ps = append(ps, strconv.Itoa(p))
 91+		}
 92+		hostPorts = strings.Join(ps, ",")
 93+	}
 94+	return []string{s.Pasta, "--config-net", "--quiet", "-t", "none", "-u", "none", "-T", hostPorts, "-U", "none"}
 95+}
 96+
 97 func (s *Systemd) Alive(handle string) bool {
 98 	unit := strings.TrimPrefix(handle, "unit:")
 99 	out, _ := exec.Command("systemctl", "show", "-p", "ActiveState", "--value", unit+".service").Output()
100@@ -202,6 +237,31 @@ func (s *Systemd) Stop(handle string) error {
101 	return exec.Command("systemctl", "stop", "--no-block", strings.TrimPrefix(handle, "unit:")+".service").Run()
102 }
103 
104+// chownTree hands paths (recursively) to user, unless we aren't root.
105+func chownTree(name string, paths ...string) error {
106+	if os.Geteuid() != 0 {
107+		return nil
108+	}
109+	u, err := user.Lookup(name)
110+	if err != nil {
111+		return err
112+	}
113+	uid, _ := strconv.Atoi(u.Uid)
114+	gid, _ := strconv.Atoi(u.Gid)
115+	for _, p := range paths {
116+		err := filepath.WalkDir(p, func(path string, d fs.DirEntry, err error) error {
117+			if err != nil {
118+				return err
119+			}
120+			return os.Lchown(path, uid, gid)
121+		})
122+		if err != nil {
123+			return err
124+		}
125+	}
126+	return nil
127+}
128+
129 func orDefault(s, d string) string {
130 	if s == "" {
131 		return d

 1@@ -300,16 +300,20 @@ func TestReconcile(t *testing.T) {
 2 
 3 func TestSystemdCommand(t *testing.T) {
 4 	s := &Systemd{Engine: "/bin/ci", LocksDir: "/var/lib/ci/locks", CacheDir: "/var/lib/ci/cache",
 5-		Identity: "/run/keys/ci-age", Pasta: "/bin/pasta", Properties: []string{"MemoryMax=8G"}, Env: []string{"CI_CLONE_URL=git@h:{repo}"}}
 6+		Credentials: map[string]string{"age-identity": "/run/keys/ci-age", "ssh-key": "/run/keys/ci-ssh"}, Pasta: "/bin/pasta", Properties: []string{"MemoryMax=8G"}, Env: []string{"CI_CLONE_URL=git@h:{repo}"}}
 7 	argv := strings.Join(s.Command(Job{Run: "r1-deploy", Dir: "/var/lib/ci/runs/r1-deploy", Repo: "sovrn", Segment: 2, Args: []string{"--choose", "1"}}), " ")
 8 	for _, want := range []string{
 9 		"--unit=ci-r1-deploy-2", "--slice=ci.slice", "--uid=ci", "ProtectSystem=strict", "PrivateTmp=yes",
10 		"ReadWritePaths=/var/lib/ci/runs/r1-deploy /var/lib/ci/cache/home/sovrn /var/lib/ci/locks /nix/var/nix/daemon-socket",
11-		"LoadCredential=age-identity:/run/keys/ci-age", "--setenv=HOME=/var/lib/ci/cache/home/sovrn", "MemoryMax=8G",
12+		"LoadCredential=age-identity:/run/keys/ci-age", "LoadCredential=ssh-key:/run/keys/ci-ssh", "--setenv=HOME=/var/lib/ci/cache/home/sovrn", "MemoryMax=8G",
13 		"-- /bin/pasta --config-net --quiet -t none -u none -T none -U none -- /bin/ci runner-exec --shell devenv --locks-dir /var/lib/ci/locks --choose 1 /var/lib/ci/runs/r1-deploy",
14 	} {
15 		if !strings.Contains(argv, want) {
16 			t.Errorf("command lacks %q:\n%s", want, argv)
17 		}
18 	}
19+	s.PastaHostPorts = []int{23231, 22}
20+	if p := strings.Join(s.PastaCommand(), " "); p != "/bin/pasta --config-net --quiet -t none -u none -T 23231,22 -U none" {
21+		t.Errorf("pasta: %s", p)
22+	}
23 }

  1@@ -0,0 +1,446 @@
  2+# NixOS module: services.ci.{bridge,runner,appview} (DESIGN.md §5, §10, §11).
  3+#
  4+#   bridge   the forge side: `ci bridge serve`, plus hookCommand for the
  5+#            forge's post-receive hook (it spools; the bridge ingests)
  6+#   runner   `ci runner serve` as root (it starts sandboxed transient units
  7+#            for the `ci` user in ci.slice), the `ci` user and the slice
  8+#   appview  `ci appview serve` (put it behind a TLS reverse proxy)
  9+#
 10+# Secrets (tokens, the runner's age identity and SSH key, the password hash)
 11+# are paths to files, e.g. Colmena keys; none of them enters the store.
 12+self:
 13+{
 14+  config,
 15+  lib,
 16+  pkgs,
 17+  ...
 18+}:
 19+
 20+let
 21+  inherit (lib)
 22+    mkEnableOption
 23+    mkIf
 24+    mkMerge
 25+    mkOption
 26+    optionals
 27+    types
 28+    ;
 29+  cfg = config.services.ci;
 30+  exe = lib.getExe cfg.package;
 31+  nullOrPath = mkOption {
 32+    type = types.nullOr types.str;
 33+    default = null;
 34+  };
 35+  flag = name: value: optionals (value != null) [ "--${name}=${toString value}" ];
 36+  hardening = {
 37+    NoNewPrivileges = true;
 38+    PrivateTmp = true;
 39+    ProtectHome = true;
 40+    ProtectSystem = "strict";
 41+    ProtectKernelTunables = true;
 42+    ProtectKernelModules = true;
 43+    ProtectControlGroups = true;
 44+    RestrictSUIDSGID = true;
 45+    LockPersonality = true;
 46+  };
 47+  # SSH for segments (git, colmena, nix copy, ssh): OpenSSH reads the passwd
 48+  # home's ~/.ssh/config, so this is the ci user's, and ${VAR} expands per
 49+  # process: each unit uses its own credential. The forge on localhost is
 50+  # trusted on first use; everything else must be in knownHostsFile.
 51+  sshConfig = pkgs.writeText "ci-ssh-config" ''
 52+    Host localhost 127.0.0.1 ::1
 53+      StrictHostKeyChecking accept-new
 54+      UserKnownHostsFile ''${HOME}/.ssh/known_hosts.local
 55+    Host *
 56+      IdentityFile ''${CREDENTIALS_DIRECTORY}/ssh-key
 57+      IdentitiesOnly yes
 58+      UserKnownHostsFile ${cfg.runner.knownHostsFile}
 59+      StrictHostKeyChecking yes
 60+  '';
 61+  # What segments find on PATH (plus the engine's own directory, first).
 62+  segmentPath = pkgs.buildEnv {
 63+    name = "ci-segment-path";
 64+    paths = [
 65+      cfg.package
 66+      pkgs.bashInteractive
 67+      pkgs.coreutils
 68+      pkgs.findutils
 69+      pkgs.gnugrep
 70+      pkgs.gnused
 71+      pkgs.gawk
 72+      pkgs.git
 73+      pkgs.openssh
 74+      pkgs.jq
 75+      pkgs.curl
 76+      pkgs.just
 77+      config.nix.package
 78+    ]
 79+    ++ optionals (cfg.runner.shell == "devenv") [ cfg.runner.devenvPackage ]
 80+    ++ cfg.runner.extraPackages;
 81+  };
 82+in
 83+{
 84+  options.services.ci = {
 85+    package = mkOption {
 86+      type = types.package;
 87+      default = self.packages.${pkgs.stdenv.hostPlatform.system}.ci;
 88+      description = "The ci package (engine and hosts).";
 89+    };
 90+
 91+    bridge = {
 92+      enable = mkEnableOption "the ci forge bridge (knot-compatible /events)";
 93+      listen = mkOption {
 94+        type = types.str;
 95+        default = "127.0.0.1:8475";
 96+      };
 97+      forgeHost = mkOption {
 98+        type = types.str;
 99+        description = "Forge host name; repos are did:web:<forgeHost>:<repo>.";
100+      };
101+      ownerDid = nullOrPath // {
102+        description = "DID of the repos' owner.";
103+      };
104+      tokenFile = nullOrPath // {
105+        description = "Bearer token required on /events and /ingest.";
106+      };
107+      spoolDir = mkOption {
108+        type = types.str;
109+        default = "/var/lib/ci-bridge/spool";
110+        description = "Where the forge's hook spools pushes (the hook runs as the forge's user; root on the forge).";
111+      };
112+      hookCommand = mkOption {
113+        type = types.str;
114+        readOnly = true;
115+        description = "Command for the forge's post-receive hook (stdin: git's ref lines).";
116+        default =
117+          "${exe} bridge notify --spool ${cfg.bridge.spoolDir} --server http://${cfg.bridge.listen}"
118+          + lib.optionalString (cfg.bridge.tokenFile != null) " --token-file ${cfg.bridge.tokenFile}";
119+      };
120+    };
121+
122+    runner = {
123+      enable = mkEnableOption "the ci runner (spindle-compatible host for the engine)";
124+      listen = mkOption {
125+        type = types.str;
126+        default = "127.0.0.1:8476";
127+      };
128+      stateDir = mkOption {
129+        type = types.str;
130+        default = "/var/lib/ci";
131+      };
132+      bridgeUrl = mkOption {
133+        type = types.str;
134+        default = "ws://${cfg.bridge.listen}/events";
135+        description = "The bridge's /events websocket.";
136+      };
137+      bridgeTokenFile = nullOrPath;
138+      tokenFile = nullOrPath // {
139+        description = "Bearer token required by the runner's API (give the same file to the appview).";
140+      };
141+      knot = mkOption {
142+        type = types.str;
143+        default = cfg.bridge.forgeHost;
144+        description = "The forge host.";
145+      };
146+      ownerDid = nullOrPath;
147+      cloneUrl = mkOption {
148+        type = types.str;
149+        example = "ssh://git@localhost:23231/{repo}";
150+        description = "Clone URL template with {repo} (for mirrors and segments; pushes go there too).";
151+      };
152+      maxConcurrent = mkOption {
153+        type = types.ints.positive;
154+        default = 2;
155+      };
156+      shell = mkOption {
157+        type = types.enum [
158+          "devenv"
159+          "bash"
160+        ];
161+        default = "devenv";
162+        description = "How actions run (bash: without devenv, for tests).";
163+      };
164+      devenvPackage = mkOption {
165+        type = types.package;
166+        default = pkgs.devenv;
167+      };
168+      extraPackages = mkOption {
169+        type = types.listOf types.package;
170+        default = [ ];
171+        description = "More tools on segments' PATH (each repo's devenv brings its own).";
172+      };
173+      identityFile = nullOrPath // {
174+        description = "age identity for the secrets vault (loaded into segments as a credential).";
175+      };
176+      sshKeyFile = nullOrPath // {
177+        description = "SSH key for the runner's fetches and segments' git/colmena/ssh (forge push access, fleet root).";
178+      };
179+      knownHostsFile = mkOption {
180+        type = types.str;
181+        default = "/etc/ssh/ssh_known_hosts";
182+        description = "known_hosts for segments' SSH (programs.ssh.knownHosts writes the default).";
183+      };
184+      pasta = mkOption {
185+        type = types.bool;
186+        default = false;
187+        description = ''
188+          Give each segment its own network namespace (egress through pasta), so
189+          dev services can't clash with the host's ports. Caveats: commands run
190+          as uid 0 of a user namespace (mapped to the ci user; tools that refuse
191+          root, like postgres' initdb, won't run), root-owned files look
192+          unowned there so OpenSSH refuses its config (no SSH: no deploys), and
193+          the hostname reads pasta-<host>. For test-only workflows; off by
194+          default. See DESIGN.md §10.
195+        '';
196+      };
197+      pastaHostPorts = mkOption {
198+        type = types.listOf types.port;
199+        default = [ ];
200+        example = [ 23231 ];
201+        description = "With pasta: TCP ports of a segment's localhost forwarded to the host's (the forge's SSH port, for a localhost cloneUrl).";
202+      };
203+      user = mkOption {
204+        type = types.str;
205+        default = "ci";
206+        description = "Unprivileged user segments run as.";
207+      };
208+      selfHost = mkOption {
209+        type = types.str;
210+        default = config.networking.fqdnOrHostName;
211+        description = "This host's name in the fleet, as CI_SELF_HOST (deploy recipes put it last).";
212+      };
213+      slice = {
214+        cpuWeight = mkOption {
215+          type = types.int;
216+          default = 20;
217+          description = "CPUWeight of ci.slice (default 100 elsewhere): services win.";
218+        };
219+        ioWeight = mkOption {
220+          type = types.int;
221+          default = 20;
222+        };
223+        memoryMax = mkOption {
224+          type = types.nullOr types.str;
225+          default = null;
226+          example = "6G";
227+        };
228+      };
229+      devenvCache = mkOption {
230+        type = types.bool;
231+        default = true;
232+        description = "Add devenv.cachix.org as a substituter (segments aren't trusted users and can't add it).";
233+      };
234+      ntfy = {
235+        url = nullOrPath // {
236+          description = "ntfy topic URL for gates and failures.";
237+        };
238+        tokenFile = nullOrPath;
239+        success = mkOption {
240+          type = types.bool;
241+          default = false;
242+        };
243+      };
244+      appviewUrl = nullOrPath // {
245+        description = "Linked from notifications.";
246+      };
247+    };
248+
249+    appview = {
250+      enable = mkEnableOption "the ci appview (web UI and agent API)";
251+      listen = mkOption {
252+        type = types.str;
253+        default = "127.0.0.1:8477";
254+      };
255+      runnerUrl = mkOption {
256+        type = types.str;
257+        default = "http://${cfg.runner.listen}";
258+      };
259+      runnerTokenFile = mkOption {
260+        type = types.nullOr types.str;
261+        default = cfg.runner.tokenFile;
262+      };
263+      passwordHashFile = mkOption {
264+        type = types.str;
265+        description = "bcrypt hash of the owner's password (mkpasswd -m bcrypt -R 12).";
266+      };
267+      sessionKeyFile = nullOrPath;
268+      agentTokensFile = nullOrPath // {
269+        description = ''Lines of "<name> <token>" for agents.'';
270+      };
271+      knot = mkOption {
272+        type = types.str;
273+        default = cfg.runner.knot;
274+      };
275+      ownerDid = mkOption {
276+        type = types.nullOr types.str;
277+        default = cfg.runner.ownerDid;
278+      };
279+      secureCookies = mkOption {
280+        type = types.bool;
281+        default = true;
282+      };
283+    };
284+  };
285+
286+  config = mkMerge [
287+    (mkIf cfg.bridge.enable {
288+      users.users.ci-bridge = {
289+        isSystemUser = true;
290+        group = "ci-bridge";
291+      };
292+      users.groups.ci-bridge = { };
293+      # the hook (root on the forge) writes files here; the bridge reads and
294+      # deletes them
295+      systemd.tmpfiles.rules = [ "d ${cfg.bridge.spoolDir} 0755 ci-bridge ci-bridge -" ];
296+      systemd.services.ci-bridge = {
297+        description = "ci forge bridge";
298+        wantedBy = [ "multi-user.target" ];
299+        after = [ "network.target" ];
300+        serviceConfig = hardening // {
301+          User = "ci-bridge";
302+          StateDirectory = "ci-bridge";
303+          ReadWritePaths = [ cfg.bridge.spoolDir ];
304+          ExecStart = lib.escapeShellArgs (
305+            [
306+              exe
307+              "bridge"
308+              "serve"
309+              "--listen=${cfg.bridge.listen}"
310+              "--db=/var/lib/ci-bridge/bridge.db"
311+              "--spool=${cfg.bridge.spoolDir}"
312+              "--forge-host=${cfg.bridge.forgeHost}"
313+            ]
314+            ++ flag "owner-did" cfg.bridge.ownerDid
315+            ++ flag "token-file" cfg.bridge.tokenFile
316+          );
317+          Restart = "always";
318+          RestartSec = 5;
319+        };
320+      };
321+    })
322+
323+    (mkIf cfg.runner.enable {
324+      users.users.${cfg.runner.user} = {
325+        isSystemUser = true;
326+        group = cfg.runner.user;
327+        home = "${cfg.runner.stateDir}/cache/home";
328+      };
329+      users.groups.${cfg.runner.user} = { };
330+      systemd.tmpfiles.rules = [
331+        "d ${cfg.runner.stateDir}/cache/home 0755 ${cfg.runner.user} ${cfg.runner.user} -"
332+        "d ${cfg.runner.stateDir}/cache/home/.ssh 0700 ${cfg.runner.user} ${cfg.runner.user} -"
333+        "L+ ${cfg.runner.stateDir}/cache/home/.ssh/config - - - - ${sshConfig}"
334+      ];
335+      systemd.slices.ci = {
336+        description = "ci segments";
337+        sliceConfig = {
338+          CPUWeight = cfg.runner.slice.cpuWeight;
339+          IOWeight = cfg.runner.slice.ioWeight;
340+        }
341+        // lib.optionalAttrs (cfg.runner.slice.memoryMax != null) { MemoryMax = cfg.runner.slice.memoryMax; };
342+      };
343+      nix.settings = mkIf cfg.runner.devenvCache {
344+        extra-substituters = [ "https://devenv.cachix.org" ];
345+        extra-trusted-public-keys = [ "devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw=" ];
346+      };
347+      systemd.services.ci-runner = {
348+        description = "ci runner";
349+        wantedBy = [ "multi-user.target" ];
350+        after = [
351+          "network.target"
352+          "ci-bridge.service"
353+        ];
354+        path = [
355+          pkgs.git
356+          pkgs.openssh
357+          pkgs.systemd
358+        ];
359+        environment = lib.optionalAttrs (cfg.runner.sshKeyFile != null) {
360+          GIT_SSH_COMMAND = "ssh -i ${cfg.runner.sshKeyFile} -o IdentitiesOnly=yes -o UserKnownHostsFile=${cfg.runner.stateDir}/known_hosts -o StrictHostKeyChecking=accept-new";
361+        };
362+        # A runner restart (or a self-deploy) must not kill segments: they
363+        # are transient units of their own, not children of this service.
364+        restartIfChanged = true;
365+        serviceConfig = {
366+          # root: it starts units for the ci user (granting ci polkit's
367+          # manage-units would let a segment start root units)
368+          StateDirectory = "ci";
369+          ExecStart = lib.escapeShellArgs (
370+            [
371+              exe
372+              "runner"
373+              "serve"
374+              "--listen=${cfg.runner.listen}"
375+              "--state-dir=${cfg.runner.stateDir}"
376+              "--bridge=${cfg.runner.bridgeUrl}"
377+              "--knot=${cfg.runner.knot}"
378+              "--clone-url=${cfg.runner.cloneUrl}"
379+              "--max-concurrent=${toString cfg.runner.maxConcurrent}"
380+              "--launcher=systemd"
381+              "--shell=${cfg.runner.shell}"
382+              "--slice=ci.slice"
383+              "--user=${cfg.runner.user}"
384+              "--setenv=PATH=${segmentPath}/bin"
385+              "--setenv=CI_SSH_KNOWN_HOSTS=${cfg.runner.knownHostsFile}"
386+              "--setenv=NIX_REMOTE=daemon"
387+              "--setenv=CI_SELF_HOST=${cfg.runner.selfHost}"
388+            ]
389+            ++ flag "owner-did" cfg.runner.ownerDid
390+            ++ flag "bridge-token-file" cfg.runner.bridgeTokenFile
391+            ++ flag "token-file" cfg.runner.tokenFile
392+            ++ flag "identity" cfg.runner.identityFile
393+            ++ flag "ssh-key" cfg.runner.sshKeyFile
394+            ++ optionals cfg.runner.pasta [ "--pasta=${pkgs.passt}/bin/pasta" ]
395+            ++ optionals (cfg.runner.pasta && cfg.runner.pastaHostPorts != [ ]) [
396+              "--pasta-host-ports=${lib.concatMapStringsSep "," toString cfg.runner.pastaHostPorts}"
397+            ]
398+            ++ flag "ntfy" cfg.runner.ntfy.url
399+            ++ flag "ntfy-token-file" cfg.runner.ntfy.tokenFile
400+            ++ optionals cfg.runner.ntfy.success [ "--ntfy-success" ]
401+            ++ flag "appview-url" cfg.runner.appviewUrl
402+          );
403+          Restart = "always";
404+          RestartSec = 5;
405+          ProtectHome = true;
406+          PrivateTmp = true;
407+        };
408+      };
409+    })
410+
411+    (mkIf cfg.appview.enable {
412+      systemd.services.ci-appview = {
413+        description = "ci appview";
414+        wantedBy = [ "multi-user.target" ];
415+        after = [ "ci-runner.service" ];
416+        serviceConfig = hardening // {
417+          DynamicUser = true;
418+          # DynamicUser can't read root-only key files; load them as credentials
419+          LoadCredential = [
420+            "password-hash:${cfg.appview.passwordHashFile}"
421+          ]
422+          ++ optionals (cfg.appview.runnerTokenFile != null) [ "runner-token:${cfg.appview.runnerTokenFile}" ]
423+          ++ optionals (cfg.appview.sessionKeyFile != null) [ "session-key:${cfg.appview.sessionKeyFile}" ]
424+          ++ optionals (cfg.appview.agentTokensFile != null) [ "agent-tokens:${cfg.appview.agentTokensFile}" ];
425+          ExecStart = lib.escapeShellArgs (
426+            [
427+              exe
428+              "appview"
429+              "serve"
430+              "--listen=${cfg.appview.listen}"
431+              "--runner=${cfg.appview.runnerUrl}"
432+              "--password-hash-file=%d/password-hash"
433+              "--knot=${cfg.appview.knot}"
434+              "--secure-cookies=${lib.boolToString cfg.appview.secureCookies}"
435+            ]
436+            ++ optionals (cfg.appview.runnerTokenFile != null) [ "--runner-token-file=%d/runner-token" ]
437+            ++ optionals (cfg.appview.sessionKeyFile != null) [ "--session-key-file=%d/session-key" ]
438+            ++ optionals (cfg.appview.agentTokensFile != null) [ "--agent-tokens-file=%d/agent-tokens" ]
439+            ++ flag "owner-did" cfg.appview.ownerDid
440+          );
441+          Restart = "always";
442+          RestartSec = 5;
443+        };
444+      };
445+    })
446+  ];
447+}

  1@@ -0,0 +1,206 @@
  2+# End to end on one NixOS VM, as on the forge host: soft-serve (as root,
  3+# like the forge role) with the bridge's post-receive hook, and the bridge,
  4+# runner (systemd launcher) and appview from nix/module.nix. A push runs a
  5+# workflow in a sandboxed transient unit as the ci user, pauses at a gate,
  6+# and finishes after the gate is approved through the runner's API.
  7+{ self, pkgs }:
  8+
  9+let
 10+  # test-only keys, generated at build time
 11+  keys = pkgs.runCommand "ci-test-keys" { nativeBuildInputs = [ pkgs.openssh pkgs.age ]; } ''
 12+    mkdir $out
 13+    ssh-keygen -q -t ed25519 -N "" -C ci-test -f $out/ssh-key
 14+    age-keygen -o $out/age-identity 2>/dev/null
 15+  '';
 16+  token = pkgs.writeText "token" "test-token-0123456789";
 17+  passwordHash = pkgs.runCommand "pw" { nativeBuildInputs = [ pkgs.mkpasswd ]; } ''
 18+    mkpasswd -m bcrypt -R 5 pw > $out
 19+  '';
 20+  workflow = pkgs.writeText "deploy.yml" ''
 21+    when: [{ event: push, branch: main }]
 22+    engine: devenv
 23+    steps: [build, deploy, record]
 24+    actions:
 25+      build:
 26+        command: |
 27+          echo "user=$(id -un)"
 28+          if touch /etc/ci-test 2>/dev/null; then echo etc-writable; else echo etc-readonly; fi
 29+          echo "identity-lines=$(grep -c ^AGE-SECRET-KEY- "$SECRETS_IDENTITY")"
 30+          echo "home=$HOME"
 31+          echo "unit=$(cat /proc/self/cgroup)"
 32+          # the appview holds 127.0.0.1:8477 on the host; a segment's own
 33+          # network namespace has the port free
 34+          timeout 2 ci bridge serve --listen 127.0.0.1:8477 --forge-host x --db "$CI_RUN_DIR/x.db" --spool "$CI_RUN_DIR/sp" 2>/dev/null || echo "port-isolated rc=$?"
 35+          ci-result ok --summary "built on $(cat /proc/sys/kernel/hostname)"
 36+      deploy:
 37+        command: echo deploying
 38+        gate: { prompt: "Deploy demo?" }
 39+      record:
 40+        command: |
 41+          git -c user.name=ci -c user.email=ci@test commit -q --allow-empty -m "recorded by ci"
 42+          git push -q origin HEAD:refs/heads/recorded
 43+          echo pushed
 44+  '';
 45+  hook = pkgs.writeShellScript "post-receive" ''
 46+    exec ${self.packages.${pkgs.stdenv.hostPlatform.system}.ci}/bin/ci bridge notify \
 47+      --spool /var/lib/ci-bridge/spool --server http://127.0.0.1:8475 --token-file ${token}
 48+  '';
 49+  node =
 50+    { pasta }:
 51+    { pkgs, ... }:
 52+    {
 53+      imports = [ self.nixosModules.default ];
 54+      virtualisation.memorySize = 2048;
 55+      environment.systemPackages = [ pkgs.git pkgs.jq pkgs.curl pkgs.openssh ];
 56+      environment.etc."ci/ssh-key" = {
 57+        source = "${keys}/ssh-key";
 58+        mode = "0600";
 59+      };
 60+      environment.etc."ci/age-identity" = {
 61+        source = "${keys}/age-identity";
 62+        mode = "0600";
 63+      };
 64+
 65+      # soft-serve as the forge runs it (root, /var/soft/data, a custom hook)
 66+      systemd.services.soft-serve = {
 67+        wantedBy = [ "multi-user.target" ];
 68+        after = [ "network.target" ];
 69+        environment = {
 70+          SOFT_SERVE_DATA_PATH = "/var/soft/data";
 71+          SOFT_SERVE_INITIAL_ADMIN_KEYS = builtins.readFile "${keys}/ssh-key.pub";
 72+          SOFT_SERVE_SSH_LISTEN_ADDR = ":23231";
 73+        };
 74+        preStart = ''
 75+          mkdir -p /var/soft/data/hooks
 76+          ln -sf ${hook} /var/soft/data/hooks/post-receive
 77+        '';
 78+        serviceConfig.ExecStart = "${pkgs.soft-serve}/bin/soft serve";
 79+        path = [ pkgs.git ];
 80+      };
 81+
 82+      services.ci = {
 83+        bridge = {
 84+          enable = true;
 85+          forgeHost = "git.test";
 86+          ownerDid = "did:plc:test";
 87+          tokenFile = "${token}";
 88+        };
 89+        runner = {
 90+          enable = true;
 91+          shell = "bash";
 92+          inherit pasta;
 93+          pastaHostPorts = [ 23231 ];
 94+          cloneUrl = "ssh://git@localhost:23231/{repo}";
 95+          bridgeTokenFile = "${token}";
 96+          tokenFile = "${token}";
 97+          identityFile = "/etc/ci/age-identity";
 98+          sshKeyFile = "/etc/ci/ssh-key";
 99+          devenvCache = false;
100+        };
101+        appview = {
102+          enable = true;
103+          passwordHashFile = "${passwordHash}";
104+          secureCookies = false;
105+        };
106+      };
107+    };
108+in
109+pkgs.testers.runNixOSTest {
110+  name = "ci-e2e";
111+  # forge: the default configuration (what deploys); pastaforge: the same
112+  # with each segment in its own network namespace
113+  nodes.forge = node { pasta = false; };
114+  nodes.pastaforge = node { pasta = true; };
115+
116+  testScript = ''
117+    import json
118+    auth = "-H 'Authorization: Bearer test-token-0123456789'"
119+    api = f"curl -sf {auth} http://127.0.0.1:8476"
120+
121+    def dump(m):
122+        print(m.execute(f"{api}/api/runs | jq '.[0] | .status, .state.status, .state.end'")[1])
123+        print(m.execute(f"{api}/api/runs | jq -r '.[0].id' | xargs -I{{}} curl -s {auth} http://127.0.0.1:8476/api/runs/{{}}/log.txt")[1])
124+        print(m.execute("tail -n 40 /var/lib/ci/runs/*/segment-*.log")[1])
125+        print(m.execute("journalctl -u ci-runner -u 'ci-*-1.service' -u 'ci-*-2.service' -n 60 --no-pager")[1])
126+
127+    def wait_status(m, cmd, timeout):
128+        try:
129+            m.wait_until_succeeds(cmd, timeout=timeout)
130+        except Exception:
131+            dump(m)
132+            raise
133+
134+    def flow(m, pasta):
135+        m.wait_for_unit("soft-serve.service")
136+        m.wait_for_open_port(23231)
137+        for u in ["ci-bridge", "ci-runner", "ci-appview"]:
138+            m.wait_for_unit(f"{u}.service")
139+        m.wait_for_open_port(8477)
140+        m.succeed("""
141+          set -e
142+          mkdir -p /tmp/w/.tangled/workflows && cd /tmp/w
143+          git init -q -b main
144+          cp ${workflow} .tangled/workflows/deploy.yml
145+          git add -A && git -c user.name=t -c user.email=t@t commit -qm init
146+          GIT_SSH_COMMAND='ssh -i /etc/ci/ssh-key -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/tmp/kh' \
147+            git push -q ssh://git@localhost:23231/demo main
148+        """)
149+
150+        with subtest(f"{m.name}: the push runs the workflow and pauses at the gate"):
151+            wait_status(m, f"{api}/api/runs | jq -e '.[0].status == \"paused\"'", 180)
152+            runs = json.loads(m.succeed(f"{api}/api/runs"))
153+            run = runs[0]["id"]
154+            assert runs[0]["state"]["gate"]["prompt"] == "Deploy demo?", runs[0]["state"]
155+
156+        with subtest(f"{m.name}: segments run sandboxed in ci.slice"):
157+            text = m.succeed(f"{api}/api/runs/{run}/log.txt")
158+            print(text)
159+            assert "etc-readonly" in text, text
160+            assert "identity-lines=1" in text, text
161+            assert "ci.slice" in text and f"ci-{run}-1.service" in text, text
162+            if pasta:
163+                # its own network namespace: the host's port 8477 is free there
164+                assert "port-isolated rc=124" in text, text
165+                # caveat: uid 0 of pasta's user namespace, hostname pasta-<host>
166+                assert "user=root" in text and "built on pasta-" in text, text
167+            else:
168+                assert "user=ci" in text, text
169+                assert f"built on {m.name}" in text, text
170+                assert "port-isolated rc=1" in text, text  # shares the host network: 8477 is taken
171+
172+        if pasta:
173+            # caveat: in pasta's user namespace root-owned files look unowned,
174+            # and OpenSSH refuses its config: no SSH (so no deploy) under pasta
175+            with subtest(f"{m.name}: dismissing the gate cancels the run"):
176+                m.succeed(f"{api}/api/runs/{run}/gate -X POST -d '{{\"dismiss\":true,\"actor\":\"did:plc:test\"}}'")
177+                wait_status(m, f"{api}/api/runs/{run} | jq -e '.status == \"cancelled\"'", 120)
178+            return run
179+
180+        with subtest(f"{m.name}: approving the gate finishes the run, pushing with the ssh key"):
181+            m.succeed(f"{api}/api/runs/{run}/gate -X POST -d '{{\"option\":1,\"actor\":\"did:plc:test\"}}'")
182+            wait_status(m, f"{api}/api/runs/{run} | jq -e '.status == \"success\"'", 120)
183+            assert "pushed" in m.succeed(f"{api}/api/runs/{run}/log.txt")
184+            m.succeed("git --git-dir=/var/soft/data/repos/demo.git rev-parse --verify refs/heads/recorded")
185+        return run
186+
187+    start_all()
188+    run = flow(forge, pasta=False)
189+    flow(pastaforge, pasta=True)
190+
191+    with subtest("records: pipeline, statuses and gate on the runner's /events"):
192+        out = forge.succeed("${pkgs.sqlite}/bin/sqlite3 /var/lib/ci/runner.db 'select nsid, event from events order by created'")
193+        assert "sh.tangled.pipeline|" in out, out
194+        statuses = [l for l in out.splitlines() if l.startswith("sh.tangled.pipeline.status|")]
195+        assert [json.loads(l.split("|", 1)[1])["status"] for l in statuses] == ["pending", "running", "success"], statuses
196+        assert "run.rtw.ci.gate|" in out, out
197+
198+    with subtest("the appview is up and asks for a login"):
199+        forge.succeed("curl -sf http://127.0.0.1:8477/healthz")
200+        forge.succeed("curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8477/ | grep -q 303")
201+
202+    with subtest("a runner restart keeps finished state"):
203+        forge.succeed("systemctl restart ci-runner.service")
204+        forge.wait_for_open_port(8476)
205+        forge.succeed(f"{api}/api/runs/{run} | jq -e '.status == \"success\"'")
206+  '';
207+}