e2e.nix
1# End to end on one NixOS VM, as on the forge host: soft-serve (as root,
2# like the forge role) with the bridge's post-receive hook, and the bridge,
3# runner (systemd launcher) and appview from nix/module.nix. A push runs a
4# workflow in a sandboxed transient unit as the ci user, pauses at a gate,
5# and finishes after the gate is approved through the runner's API.
6{ self, pkgs }:
7
8let
9 # test-only keys, generated at build time
10 keys = pkgs.runCommand "ci-test-keys" { nativeBuildInputs = [ pkgs.openssh pkgs.age ]; } ''
11 mkdir $out
12 ssh-keygen -q -t ed25519 -N "" -C ci-test -f $out/ssh-key
13 age-keygen -o $out/age-identity 2>/dev/null
14 '';
15 token = pkgs.writeText "token" "test-token-0123456789";
16 passwordHash = pkgs.runCommand "pw" { nativeBuildInputs = [ pkgs.mkpasswd ]; } ''
17 mkpasswd -m bcrypt -R 5 pw > $out
18 '';
19 workflow = pkgs.writeText "deploy.yml" ''
20 when: [{ event: push, branch: main }]
21 engine: devenv
22 steps: [build, deploy, record]
23 actions:
24 build:
25 command: |
26 echo "user=$(id -un)"
27 if touch /etc/ci-test 2>/dev/null; then echo etc-writable; else echo etc-readonly; fi
28 echo "identity-lines=$(grep -c ^AGE-SECRET-KEY- "$SECRETS_IDENTITY")"
29 echo "home=$HOME"
30 echo "unit=$(cat /proc/self/cgroup)"
31 # the appview holds 127.0.0.1:8477 on the host; a segment's own
32 # network namespace has the port free
33 timeout 2 ci bridge serve --listen 127.0.0.1:8477 --forge-host x --db "$CI_RUN_DIR/x.db" --spool "$CI_RUN_DIR/sp" 2>/dev/null || echo "port-isolated rc=$?"
34 ci-result ok --summary "built on $(cat /proc/sys/kernel/hostname)"
35 deploy:
36 command: echo deploying
37 gate: { prompt: "Deploy demo?" }
38 record:
39 command: |
40 git -c user.name=ci -c user.email=ci@test commit -q --allow-empty -m "recorded by ci"
41 git push -q origin HEAD:refs/heads/recorded
42 echo pushed
43 '';
44 hook = pkgs.writeShellScript "post-receive" ''
45 exec ${self.packages.${pkgs.stdenv.hostPlatform.system}.ci}/bin/ci bridge notify \
46 --spool /var/lib/ci-bridge/spool --server http://127.0.0.1:8475 --token-file /etc/ci/bridge-token
47 '';
48 node =
49 { pasta }:
50 { pkgs, ... }:
51 {
52 imports = [ self.nixosModules.default ];
53 virtualisation.memorySize = 2048;
54 environment.systemPackages = [ pkgs.git pkgs.jq pkgs.curl pkgs.openssh ];
55 environment.etc."ci/ssh-key" = {
56 source = "${keys}/ssh-key";
57 mode = "0600";
58 };
59 environment.etc."ci/age-identity" = {
60 source = "${keys}/age-identity";
61 mode = "0600";
62 };
63 # root-only, as the fleet uploads it: the bridge (user ci-bridge) must
64 # get it as a credential, and the forge's hook (root) reads the file
65 environment.etc."ci/bridge-token" = {
66 source = token;
67 mode = "0400";
68 };
69
70 # soft-serve as the forge runs it (root, /var/soft/data, a custom hook)
71 systemd.services.soft-serve = {
72 wantedBy = [ "multi-user.target" ];
73 after = [ "network.target" ];
74 environment = {
75 SOFT_SERVE_DATA_PATH = "/var/soft/data";
76 SOFT_SERVE_INITIAL_ADMIN_KEYS = builtins.readFile "${keys}/ssh-key.pub";
77 SOFT_SERVE_SSH_LISTEN_ADDR = ":23231";
78 };
79 preStart = ''
80 mkdir -p /var/soft/data/hooks
81 ln -sf ${hook} /var/soft/data/hooks/post-receive
82 '';
83 serviceConfig.ExecStart = "${pkgs.soft-serve}/bin/soft serve";
84 path = [ pkgs.git ];
85 };
86
87 services.ci = {
88 bridge = {
89 enable = true;
90 forgeHost = "git.test";
91 ownerDid = "did:plc:test";
92 tokenFile = "/etc/ci/bridge-token";
93 };
94 runner = {
95 enable = true;
96 shell = "bash";
97 inherit pasta;
98 pastaHostPorts = [ 23231 ];
99 cloneUrl = "ssh://git@localhost:23231/{repo}";
100 bridgeTokenFile = "${token}";
101 tokenFile = "${token}";
102 identityFile = "/etc/ci/age-identity";
103 sshKeyFile = "/etc/ci/ssh-key";
104 devenvCache = false;
105 };
106 appview = {
107 enable = true;
108 passwordHashFile = "${passwordHash}";
109 secureCookies = false;
110 };
111 };
112 };
113in
114pkgs.testers.runNixOSTest {
115 name = "ci-e2e";
116 # forge: the default configuration (what deploys); pastaforge: the same
117 # with each segment in its own network namespace
118 nodes.forge = node { pasta = false; };
119 nodes.pastaforge = node { pasta = true; };
120
121 testScript = ''
122 import json
123 auth = "-H 'Authorization: Bearer test-token-0123456789'"
124 api = f"curl -sf {auth} http://127.0.0.1:8476"
125
126 def dump(m):
127 print(m.execute(f"{api}/api/runs | jq '.[0] | .status, .state.status, .state.end'")[1])
128 print(m.execute(f"{api}/api/runs | jq -r '.[0].id' | xargs -I{{}} curl -s {auth} http://127.0.0.1:8476/api/runs/{{}}/log.txt")[1])
129 print(m.execute("tail -n 40 /var/lib/ci/runs/*/segment-*.log")[1])
130 print(m.execute("journalctl -u ci-runner -u 'ci-*-1.service' -u 'ci-*-2.service' -n 60 --no-pager")[1])
131
132 def wait_status(m, cmd, timeout):
133 try:
134 m.wait_until_succeeds(cmd, timeout=timeout)
135 except Exception:
136 dump(m)
137 raise
138
139 def flow(m, pasta):
140 m.wait_for_unit("soft-serve.service")
141 m.wait_for_open_port(23231)
142 for u in ["ci-bridge", "ci-runner", "ci-appview"]:
143 m.wait_for_unit(f"{u}.service")
144 m.wait_for_open_port(8477)
145 m.succeed("""
146 set -e
147 mkdir -p /tmp/w/.tangled/workflows && cd /tmp/w
148 git init -q -b main
149 cp ${workflow} .tangled/workflows/deploy.yml
150 git add -A && git -c user.name=t -c user.email=t@t commit -qm init
151 GIT_SSH_COMMAND='ssh -i /etc/ci/ssh-key -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/tmp/kh' \
152 git push -q ssh://git@localhost:23231/demo main
153 """)
154
155 with subtest(f"{m.name}: the push runs the workflow and pauses at the gate"):
156 wait_status(m, f"{api}/api/runs | jq -e '.[0].status == \"paused\"'", 180)
157 runs = json.loads(m.succeed(f"{api}/api/runs"))
158 run = runs[0]["id"]
159 assert runs[0]["state"]["gate"]["prompt"] == "Deploy demo?", runs[0]["state"]
160
161 with subtest(f"{m.name}: segments run sandboxed in ci.slice"):
162 text = m.succeed(f"{api}/api/runs/{run}/log.txt")
163 print(text)
164 assert "etc-readonly" in text, text
165 assert "identity-lines=1" in text, text
166 assert "ci.slice" in text and f"ci-{run}-1.service" in text, text
167 if pasta:
168 # its own network namespace: the host's port 8477 is free there
169 assert "port-isolated rc=124" in text, text
170 # caveat: uid 0 of pasta's user namespace, hostname pasta-<host>
171 assert "user=root" in text and "built on pasta-" in text, text
172 else:
173 assert "user=ci" in text, text
174 assert f"built on {m.name}" in text, text
175 assert "port-isolated rc=1" in text, text # shares the host network: 8477 is taken
176
177 if pasta:
178 # caveat: in pasta's user namespace root-owned files look unowned,
179 # and OpenSSH refuses its config: no SSH (so no deploy) under pasta
180 with subtest(f"{m.name}: dismissing the gate cancels the run"):
181 m.succeed(f"{api}/api/runs/{run}/gate -X POST -d '{{\"dismiss\":true,\"actor\":\"did:plc:test\"}}'")
182 wait_status(m, f"{api}/api/runs/{run} | jq -e '.status == \"cancelled\"'", 120)
183 return run
184
185 with subtest(f"{m.name}: approving the gate finishes the run, pushing with the ssh key"):
186 m.succeed(f"{api}/api/runs/{run}/gate -X POST -d '{{\"option\":1,\"actor\":\"did:plc:test\"}}'")
187 wait_status(m, f"{api}/api/runs/{run} | jq -e '.status == \"success\"'", 120)
188 assert "pushed" in m.succeed(f"{api}/api/runs/{run}/log.txt")
189 m.succeed("git --git-dir=/var/soft/data/repos/demo.git rev-parse --verify refs/heads/recorded")
190 return run
191
192 start_all()
193 run = flow(forge, pasta=False)
194 flow(pastaforge, pasta=True)
195
196 with subtest("records: pipeline, statuses and gate on the runner's /events"):
197 out = forge.succeed("${pkgs.sqlite}/bin/sqlite3 /var/lib/ci/runner.db 'select nsid, event from events order by created'")
198 assert "sh.tangled.pipeline|" in out, out
199 statuses = [l for l in out.splitlines() if l.startswith("sh.tangled.pipeline.status|")]
200 assert [json.loads(l.split("|", 1)[1])["status"] for l in statuses] == ["pending", "running", "success"], statuses
201 assert "run.rtw.ci.gate|" in out, out
202
203 with subtest("the appview is up and asks for a login"):
204 forge.succeed("curl -sf http://127.0.0.1:8477/healthz")
205 forge.succeed("curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8477/ | grep -q 303")
206
207 with subtest("a runner restart keeps finished state"):
208 forge.succeed("systemctl restart ci-runner.service")
209 forge.wait_for_open_port(8476)
210 forge.succeed(f"{api}/api/runs/{run} | jq -e '.status == \"success\"'")
211 '';
212}
