ci

 1@@ -362,11 +362,32 @@ Run directory
 2 
 3 Repos
 4 
 5-- `repos:` entries are cloned at the pushed revision of their default branch
 6-  (or a pinned rev) in both modes; the shas are recorded in `events.ndjson`.
 7-- Paths are relative to the main checkout. Keep the runner's layout identical
 8-  to `~/projects` so `../servers` resolves the same in both places.
 9-- `--repo name=/path` overrides a checkout locally (explicit; off by default).
10+- Layout: the main checkout is `<run>/<repo>` and each `repos:` entry is
11+  `<run>/<repo>/<path>`, which must be a sibling (`../servers` →
12+  `<run>/servers`). Locally, without `--clean`, `<run>/<repo>` is a symlink
13+  to the working copy, so uncommitted changes are visible; with `--clean`
14+  it is a fresh checkout of `--rev` (jj revset, default `@`; git rev,
15+  default `HEAD`), i.e. exactly what the VCS tracks.
16+- Extra repos are fetched into a bare mirror per repo
17+  (`$CI_CACHE_DIR/mirrors/<repo>.git`, default `~/.cache/ci`) on every run,
18+  then checked out with `git clone --shared` at a detached commit: the
19+  forge's default branch, or the entry's `rev`. Checkouts borrow the
20+  mirror's objects but are independent, so a run is deleted with `rm -rf`
21+  and nothing is written into any repository's metadata. Each checkout's
22+  `origin` is the forge URL, so recipes can push (`git push origin
23+  HEAD:main` from the detached checkout).
24+- Clone URLs are siblings of the main repo's `origin`
25+  (`[email protected]:sovrn` → `[email protected]:servers`), or
26+  `$CI_CLONE_URL` with `{repo}` (e.g. `git@host:{repo}`).
27+- `env:` on an entry exports its checkout path (e.g. `SECRETS_DIR`).
28+- Every checkout's commit is recorded as a `checkout` event in
29+  `events.ndjson`; linked directories are recorded as `linked`.
30+- `--repo name=/path` links an extra repo to a local directory instead
31+  (explicit; off by default).
32+- `ci gc [--keep 168h]` deletes finished runs older than `--keep` (never
33+  running or paused ones; symlinked working copies are only unlinked) and
34+  compacts the mirrors. Mirrors have `gc.auto=0`, so objects a live
35+  checkout borrows aren't pruned under it.
36 
37 Segments: a gate ends the process. The engine runs actions until it reaches a
38 gate, writes a `gate` event, saves `state.json` and exits with a "paused"

  1@@ -15,6 +15,7 @@ import (
  2 	"syscall"
  3 	"time"
  4 
  5+	"git.kilimanjaro.io/ci/internal/checkout"
  6 	"git.kilimanjaro.io/ci/internal/engine"
  7 	"git.kilimanjaro.io/ci/internal/lex"
  8 	"git.kilimanjaro.io/ci/internal/manifest"
  9@@ -79,16 +80,35 @@ type runFlags struct {
 10 	ref      string
 11 	maxTrans int
 12 	quiet    bool
 13+	clean    bool
 14+	rev      string
 15+	repos    repoFlag
 16+}
 17+
 18+// repoFlag collects --repo name=path overrides.
 19+type repoFlag map[string]string
 20+
 21+func (r repoFlag) String() string { return "" }
 22+func (r repoFlag) Set(v string) error {
 23+	n, p, ok := strings.Cut(v, "=")
 24+	if !ok || n == "" || p == "" {
 25+		return fmt.Errorf("want <repo>=<path>, got %q", v)
 26+	}
 27+	r[n] = p
 28+	return nil
 29 }
 30 
 31 func newRunFlags(name string) *runFlags {
 32-	f := &runFlags{fs: flag.NewFlagSet("ci "+name, flag.ContinueOnError), stubs: stubFlag{}}
 33+	f := &runFlags{fs: flag.NewFlagSet("ci "+name, flag.ContinueOnError), stubs: stubFlag{}, repos: repoFlag{}}
 34 	f.fs.Var(f.stubs, "stub", "use `action=result.json` instead of running the action (repeatable)")
 35 	f.fs.StringVar(&f.until, "until", "", "stop before this `action` runs")
 36 	f.fs.StringVar(&f.shell, "shell", "devenv", "how actions run: devenv (default) or bash (no devenv; debugging)")
 37 	f.fs.StringVar(&f.ref, "ref", "", "ref to report as CI_REF/TANGLED_REF, e.g. refs/heads/main")
 38 	f.fs.IntVar(&f.maxTrans, "max-transitions", 0, "loop guard (default 50)")
 39 	f.fs.BoolVar(&f.quiet, "q", false, "don't render events (they are still written to the run)")
 40+	f.fs.BoolVar(&f.clean, "clean", false, "run a fresh checkout of a commit (as the runner does) instead of the working copy")
 41+	f.fs.StringVar(&f.rev, "rev", "", "with --clean: the `revision` to check out (jj revset or git rev; default @ or HEAD)")
 42+	f.fs.Var(f.repos, "repo", "link an extra repo to a local directory instead of a clean clone: `name=path` (repeatable)")
 43 	return f
 44 }
 45 
 46@@ -178,17 +198,18 @@ func runCommon(name string, args []string) int {
 47 			return exitUsage
 48 		}
 49 	}
 50-	sha, dirty, err := vcs.Head(root)
 51-	if err != nil {
 52-		msg, _, _ := strings.Cut(err.Error(), "\n")
 53-		fmt.Fprintln(os.Stderr, "ci: warning: no commit for the working copy:", msg)
 54+	if f.rev != "" && !f.clean {
 55+		fmt.Fprintln(os.Stderr, "ci: --rev needs --clean")
 56+		return exitUsage
 57 	}
 58 	id := tid.New()
 59-	meta := engine.Meta{
 60-		Run: id, Workflow: m.Name, Repo: filepath.Base(root), Sha: sha, Ref: f.ref,
 61-		Kind: "manual", Host: "local", Root: root, Dirty: dirty,
 62-	}
 63 	dir := engine.Dir(filepath.Join(runsDir(), id))
 64+	meta, err := prepareLocal(m, root, dir, f)
 65+	if err != nil {
 66+		_ = os.RemoveAll(dir.Path())
 67+		fmt.Fprintln(os.Stderr, "ci:", err)
 68+		return exitFailure
 69+	}
 70 	e, err := engine.Create(m, dir, meta, ex, engine.Options{
 71 		Stubs: f.stubs, Until: f.until, NoFollow: noFollow, MaxTransitions: f.maxTrans,
 72 	})
 73@@ -223,6 +244,68 @@ func runCommon(name string, args []string) int {
 74 	return exitCode(st.Status)
 75 }
 76 
 77+// prepareLocal lays out a local run's checkouts (DESIGN.md §9 Repos): the
 78+// main checkout is the working copy (linked) or, with --clean, a clean clone
 79+// of a commit; extra repos are clean clones of their pushed revisions unless
 80+// --repo links them.
 81+func prepareLocal(m *manifest.Manifest, root string, dir engine.Dir, f *runFlags) (engine.Meta, error) {
 82+	ctx := context.Background()
 83+	repo := filepath.Base(root)
 84+	meta := engine.Meta{
 85+		Run: dir.Base(), Workflow: m.Name, Repo: repo, Ref: f.ref,
 86+		Kind: "manual", Host: "local", Root: filepath.Join(dir.Path(), repo),
 87+	}
 88+	gitDir, err := checkout.GitDir(root)
 89+	if err != nil {
 90+		return meta, err
 91+	}
 92+	origin := checkout.OriginURL(ctx, gitDir)
 93+	src := &checkout.Source{MirrorsDir: filepath.Join(cacheDir(), "mirrors"), URL: checkout.URLFunc(os.Getenv("CI_CLONE_URL"), origin)}
 94+	main := checkout.Main{Repo: repo}
 95+	if f.clean {
 96+		if meta.Sha, err = vcs.Resolve(root, f.rev); err != nil {
 97+			return meta, err
 98+		}
 99+		main.From, main.Sha, main.Origin = gitDir, meta.Sha, origin
100+	} else {
101+		main.WorkingCopy = root
102+		var err error
103+		meta.Sha, meta.Dirty, err = vcs.Head(root)
104+		if err != nil {
105+			msg, _, _ := strings.Cut(err.Error(), "\n")
106+			fmt.Fprintln(os.Stderr, "ci: warning: no commit for the working copy:", msg)
107+		}
108+	}
109+	if err := dir.Create(); err != nil {
110+		return meta, err
111+	}
112+	checked, err := checkout.Layout(ctx, src, dir.Path(), main, m.Repos, f.repos)
113+	if err != nil {
114+		return meta, err
115+	}
116+	for _, c := range checked {
117+		rev := c.Rev
118+		if c.Link {
119+			rev = "linked"
120+		}
121+		meta.Checkouts = append(meta.Checkouts, engine.Checkout{Repo: c.Repo, Path: c.Path, Rev: rev, Env: c.Env})
122+	}
123+	return meta, nil
124+}
125+
126+// cacheDir holds mirrors: $CI_CACHE_DIR, else $XDG_CACHE_HOME/ci, else
127+// ~/.cache/ci.
128+func cacheDir() string {
129+	if d := os.Getenv("CI_CACHE_DIR"); d != "" {
130+		return d
131+	}
132+	if d := os.Getenv("XDG_CACHE_HOME"); d != "" {
133+		return filepath.Join(d, "ci")
134+	}
135+	home, _ := os.UserHomeDir()
136+	return filepath.Join(home, ".cache", "ci")
137+}
138+
139 func exitCode(status string) int {
140 	switch status {
141 	case engine.StatusSuccess:

 1@@ -0,0 +1,62 @@
 2+package main
 3+
 4+import (
 5+	"flag"
 6+	"fmt"
 7+	"os"
 8+	"os/exec"
 9+	"path/filepath"
10+	"time"
11+
12+	"git.kilimanjaro.io/ci/internal/engine"
13+)
14+
15+func init() {
16+	commands["gc"] = command{"delete old finished runs and compact mirrors", cmdGC}
17+}
18+
19+func cmdGC(args []string) int {
20+	fs := flag.NewFlagSet("ci gc", flag.ContinueOnError)
21+	keep := fs.Duration("keep", 7*24*time.Hour, "keep finished runs younger than this")
22+	dry := fs.Bool("n", false, "only print what would be deleted")
23+	if err := fs.Parse(args); err != nil {
24+		return exitUsage
25+	}
26+	runs, err := listRuns()
27+	if err != nil {
28+		fmt.Fprintln(os.Stderr, "ci:", err)
29+		return exitFailure
30+	}
31+	cutoff := time.Now().Add(-*keep)
32+	removed := 0
33+	for _, d := range runs {
34+		st, err := engine.Load(d)
35+		if err != nil || !engine.Terminal(st.Status) {
36+			continue // running and paused runs are never collected
37+		}
38+		t, err := time.Parse(time.RFC3339Nano, st.Updated)
39+		if err != nil || t.After(cutoff) {
40+			continue
41+		}
42+		fmt.Printf("delete %s (%s %s/%s, %s)\n", d.Base(), st.Status, st.Meta.Repo, st.Meta.Workflow, st.Updated)
43+		if !*dry {
44+			// main checkouts of working-copy runs are symlinks: RemoveAll
45+			// removes the link, never the developer's tree
46+			if err := os.RemoveAll(d.Path()); err != nil {
47+				fmt.Fprintln(os.Stderr, "ci:", err)
48+				return exitFailure
49+			}
50+		}
51+		removed++
52+	}
53+	if !*dry {
54+		mirrors, _ := filepath.Glob(filepath.Join(cacheDir(), "mirrors", "*.git"))
55+		for _, m := range mirrors {
56+			if out, err := exec.Command("git", "-C", m, "gc", "--quiet").CombinedOutput(); err != nil {
57+				fmt.Fprintf(os.Stderr, "ci: git gc %s: %v: %s\n", m, err, out)
58+			}
59+		}
60+	}
61+	fmt.Printf("%d runs deleted\n", removed)
62+	return exitSuccess
63+}

  1@@ -0,0 +1,135 @@
  2+// Package checkout prepares the repositories of a run: the main checkout
  3+// and the manifest's extra repos, laid out as <run>/<repo> and siblings so
  4+// relative paths such as ../servers resolve the same locally and on the
  5+// runner.
  6+//
  7+// Remote repos are fetched into a bare mirror per repo (one network fetch per
  8+// run, objects shared between runs) and checked out with `git clone --shared`
  9+// at a detached commit: each checkout is independent, so a run directory can
 10+// be deleted with rm -rf, and nothing is written into the mirror's or the
 11+// developer's own repository metadata.
 12+package checkout
 13+
 14+import (
 15+	"context"
 16+	"errors"
 17+	"fmt"
 18+	"os"
 19+	"os/exec"
 20+	"path/filepath"
 21+	"strings"
 22+)
 23+
 24+// Source resolves repo names to clone URLs and keeps mirrors.
 25+type Source struct {
 26+	// MirrorsDir holds <repo>.git bare mirrors.
 27+	MirrorsDir string
 28+	// URL returns the clone URL of a repo.
 29+	URL func(repo string) (string, error)
 30+}
 31+
 32+// Mirror makes sure the mirror of repo exists and is up to date, and returns
 33+// its path. Mirrors never auto-gc, so objects a live checkout borrows through
 34+// alternates are not pruned under it; `ci gc` collects them.
 35+func (s *Source) Mirror(ctx context.Context, repo string) (string, error) {
 36+	url, err := s.URL(repo)
 37+	if err != nil {
 38+		return "", err
 39+	}
 40+	dir := filepath.Join(s.MirrorsDir, repo+".git")
 41+	if _, err := os.Stat(filepath.Join(dir, "HEAD")); errors.Is(err, os.ErrNotExist) {
 42+		if err := os.MkdirAll(s.MirrorsDir, 0o755); err != nil {
 43+			return "", err
 44+		}
 45+		tmp := dir + ".tmp"
 46+		_ = os.RemoveAll(tmp)
 47+		if _, err := git(ctx, "", "clone", "--mirror", "--quiet", url, tmp); err != nil {
 48+			return "", fmt.Errorf("mirror %s: %w", repo, err)
 49+		}
 50+		if _, err := git(ctx, tmp, "config", "gc.auto", "0"); err != nil {
 51+			return "", err
 52+		}
 53+		if err := os.Rename(tmp, dir); err != nil {
 54+			return "", err
 55+		}
 56+		return dir, nil
 57+	} else if err != nil {
 58+		return "", err
 59+	}
 60+	if _, err := git(ctx, dir, "remote", "set-url", "origin", url); err != nil {
 61+		return "", err
 62+	}
 63+	if _, err := git(ctx, dir, "fetch", "--quiet", "--prune", "origin"); err != nil {
 64+		return "", fmt.Errorf("fetch %s: %w", repo, err)
 65+	}
 66+	return dir, nil
 67+}
 68+
 69+// Resolve returns the commit rev names in the repository at gitDir. An empty
 70+// rev is HEAD (for a mirror: the forge's default branch).
 71+func Resolve(ctx context.Context, gitDir, rev string) (string, error) {
 72+	if rev == "" {
 73+		rev = "HEAD"
 74+	}
 75+	out, err := git(ctx, gitDir, "rev-parse", "--verify", "--quiet", rev+"^{commit}")
 76+	if err != nil {
 77+		return "", fmt.Errorf("no commit %q in %s", rev, gitDir)
 78+	}
 79+	return strings.TrimSpace(out), nil
 80+}
 81+
 82+// Clone checks out sha from the repository at from into dest (which must not
 83+// exist), sharing its objects. If origin is set it becomes the checkout's
 84+// origin remote, so recipes can push to the forge.
 85+func Clone(ctx context.Context, from, sha, dest, origin string) error {
 86+	if _, err := os.Lstat(dest); err == nil {
 87+		return fmt.Errorf("%s already exists", dest)
 88+	}
 89+	if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
 90+		return err
 91+	}
 92+	if _, err := git(ctx, "", "clone", "--shared", "--no-checkout", "--quiet", from, dest); err != nil {
 93+		return err
 94+	}
 95+	if _, err := git(ctx, dest, "checkout", "--quiet", "--detach", sha); err != nil {
 96+		return err
 97+	}
 98+	if origin != "" {
 99+		if _, err := git(ctx, dest, "remote", "set-url", "origin", origin); err != nil {
100+			return err
101+		}
102+	}
103+	return nil
104+}
105+
106+// GitDir returns the git directory of a local repository: .git for git and
107+// colocated jj repos, .jj/repo/store/git for jj repos without colocation.
108+func GitDir(root string) (string, error) {
109+	for _, p := range []string{filepath.Join(root, ".git"), filepath.Join(root, ".jj", "repo", "store", "git")} {
110+		if _, err := os.Stat(p); err == nil {
111+			return p, nil
112+		}
113+	}
114+	return "", fmt.Errorf("%s: no git directory (.git or .jj/repo/store/git)", root)
115+}
116+
117+// Inside reports whether p is dir or below it (lexically).
118+func Inside(dir, p string) bool {
119+	rel, err := filepath.Rel(dir, p)
120+	return err == nil && rel != ".." && !strings.HasPrefix(rel, "../")
121+}
122+
123+func git(ctx context.Context, dir string, args ...string) (string, error) {
124+	if dir != "" {
125+		args = append([]string{"-C", dir}, args...)
126+	}
127+	cmd := exec.CommandContext(ctx, "git", args...)
128+	cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
129+	var stderr strings.Builder
130+	cmd.Stderr = &stderr
131+	out, err := cmd.Output()
132+	if err != nil {
133+		return "", fmt.Errorf("git %s: %v: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
134+	}
135+	return string(out), nil
136+}

  1@@ -0,0 +1,186 @@
  2+package checkout
  3+
  4+import (
  5+	"context"
  6+	"os"
  7+	"os/exec"
  8+	"path/filepath"
  9+	"strings"
 10+	"testing"
 11+
 12+	"git.kilimanjaro.io/ci/internal/lex"
 13+)
 14+
 15+func sh(t *testing.T, dir string, args ...string) string {
 16+	t.Helper()
 17+	cmd := exec.Command(args[0], args[1:]...)
 18+	cmd.Dir = dir
 19+	cmd.Env = append(os.Environ(),
 20+		"GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t",
 21+		"GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null")
 22+	out, err := cmd.CombinedOutput()
 23+	if err != nil {
 24+		t.Fatalf("%v: %v\n%s", args, err, out)
 25+	}
 26+	return strings.TrimSpace(string(out))
 27+}
 28+
 29+// forge makes a bare repo <forge>/<name>.git with commits writing file=v1
 30+// then file=v2 on main, and a tag v1 at the first. It returns the shas.
 31+func forge(t *testing.T, dir, name string) (v1, v2 string) {
 32+	t.Helper()
 33+	work := filepath.Join(t.TempDir(), name)
 34+	if err := os.MkdirAll(work, 0o755); err != nil {
 35+		t.Fatal(err)
 36+	}
 37+	sh(t, work, "git", "init", "-q", "-b", "main")
 38+	if err := os.WriteFile(filepath.Join(work, "file"), []byte("v1"), 0o644); err != nil {
 39+		t.Fatal(err)
 40+	}
 41+	sh(t, work, "git", "add", "file")
 42+	sh(t, work, "git", "commit", "-qm", "v1")
 43+	v1 = sh(t, work, "git", "rev-parse", "HEAD")
 44+	sh(t, work, "git", "tag", "v1")
 45+	if err := os.WriteFile(filepath.Join(work, "file"), []byte("v2"), 0o644); err != nil {
 46+		t.Fatal(err)
 47+	}
 48+	sh(t, work, "git", "commit", "-qam", "v2")
 49+	v2 = sh(t, work, "git", "rev-parse", "HEAD")
 50+	bare := filepath.Join(dir, name+".git")
 51+	sh(t, dir, "git", "clone", "-q", "--bare", work, bare)
 52+	return v1, v2
 53+}
 54+
 55+func read(t *testing.T, p string) string {
 56+	t.Helper()
 57+	b, err := os.ReadFile(p)
 58+	if err != nil {
 59+		t.Fatal(err)
 60+	}
 61+	return string(b)
 62+}
 63+
 64+func TestLayout(t *testing.T) {
 65+	t.Setenv("GIT_CONFIG_GLOBAL", "/dev/null")
 66+	ctx := context.Background()
 67+	fdir := t.TempDir()
 68+	m1, m2 := forge(t, fdir, "main")
 69+	_, _ = forge(t, fdir, "servers")
 70+	d1, _ := forge(t, fdir, "data")
 71+	src := &Source{MirrorsDir: filepath.Join(t.TempDir(), "mirrors"), URL: URLFunc(filepath.Join(fdir, "{repo}.git"), "")}
 72+
 73+	run := filepath.Join(t.TempDir(), "run")
 74+	mainMirror, err := src.Mirror(ctx, "main")
 75+	if err != nil {
 76+		t.Fatal(err)
 77+	}
 78+	local := t.TempDir()
 79+	got, err := Layout(ctx, src, run, Main{Repo: "main", From: mainMirror, Sha: m1, Origin: "git@forge:main"},
 80+		[]lex.Repo{
 81+			{Repo: "servers", Path: "../servers"},
 82+			{Repo: "data", Path: "../data", Rev: "v1", Env: "SECRETS_DIR"},
 83+			{Repo: "tools", Path: "../tools"},
 84+		}, map[string]string{"tools": local})
 85+	if err != nil {
 86+		t.Fatal(err)
 87+	}
 88+	if len(got) != 4 {
 89+		t.Fatalf("%+v", got)
 90+	}
 91+	if read(t, filepath.Join(run, "main", "file")) != "v1" || got[0].Rev != m1 {
 92+		t.Errorf("main checkout not at v1: %+v", got[0])
 93+	}
 94+	if read(t, filepath.Join(run, "servers", "file")) != "v2" {
 95+		t.Error("servers not at the default branch")
 96+	}
 97+	if got[2].Rev != d1 || got[2].Env != "SECRETS_DIR" || read(t, filepath.Join(run, "data", "file")) != "v1" {
 98+		t.Errorf("data: %+v", got[2])
 99+	}
100+	if p, _ := os.Readlink(filepath.Join(run, "tools")); p != local || !got[3].Link {
101+		t.Errorf("tools override: %q %+v", p, got[3])
102+	}
103+	if u := sh(t, filepath.Join(run, "main"), "git", "remote", "get-url", "origin"); u != "git@forge:main" {
104+		t.Errorf("origin %q", u)
105+	}
106+	if u := sh(t, filepath.Join(run, "servers"), "git", "remote", "get-url", "origin"); u != filepath.Join(fdir, "servers.git") {
107+		t.Errorf("servers origin %q", u)
108+	}
109+	// the checkout borrows objects; nothing was registered in the mirror
110+	if _, err := os.Stat(filepath.Join(mainMirror, "worktrees")); err == nil {
111+		t.Error("mirror has worktree metadata")
112+	}
113+	_ = m2
114+}
115+
116+func TestMirrorFetchesNewCommits(t *testing.T) {
117+	t.Setenv("GIT_CONFIG_GLOBAL", "/dev/null")
118+	ctx := context.Background()
119+	fdir := t.TempDir()
120+	_, v2 := forge(t, fdir, "r")
121+	src := &Source{MirrorsDir: t.TempDir(), URL: URLFunc(filepath.Join(fdir, "{repo}.git"), "")}
122+	m, err := src.Mirror(ctx, "r")
123+	if err != nil {
124+		t.Fatal(err)
125+	}
126+	if sha, _ := Resolve(ctx, m, ""); sha != v2 {
127+		t.Fatalf("HEAD %s want %s", sha, v2)
128+	}
129+	// push a new commit to the forge; the next Mirror fetches it
130+	work := filepath.Join(t.TempDir(), "w")
131+	sh(t, "", "git", "clone", "-q", filepath.Join(fdir, "r.git"), work)
132+	if err := os.WriteFile(filepath.Join(work, "file"), []byte("v3"), 0o644); err != nil {
133+		t.Fatal(err)
134+	}
135+	sh(t, work, "git", "commit", "-qam", "v3")
136+	sh(t, work, "git", "push", "-q", "origin", "main")
137+	v3 := sh(t, work, "git", "rev-parse", "HEAD")
138+	if _, err := src.Mirror(ctx, "r"); err != nil {
139+		t.Fatal(err)
140+	}
141+	if sha, _ := Resolve(ctx, m, ""); sha != v3 {
142+		t.Errorf("after fetch HEAD %s want %s", sha, v3)
143+	}
144+	if _, err := Resolve(ctx, m, "nope"); err == nil {
145+		t.Error("resolved a missing rev")
146+	}
147+}
148+
149+func TestLayoutRejectsBadPaths(t *testing.T) {
150+	ctx := context.Background()
151+	src := &Source{MirrorsDir: t.TempDir(), URL: URLFunc("/nowhere/{repo}.git", "")}
152+	wc := t.TempDir()
153+	for _, p := range []string{"../../escape", "sub", "../main", "."} {
154+		run := filepath.Join(t.TempDir(), "run")
155+		_, err := Layout(ctx, src, run, Main{Repo: "main", WorkingCopy: wc}, []lex.Repo{{Repo: "x", Path: p}}, nil)
156+		if err == nil || !strings.Contains(err.Error(), "sibling") {
157+			t.Errorf("path %q: %v", p, err)
158+		}
159+	}
160+	run := filepath.Join(t.TempDir(), "run")
161+	if _, err := Layout(ctx, src, run, Main{Repo: "main", WorkingCopy: wc}, nil, map[string]string{"x": wc}); err == nil {
162+		t.Error("override for an unknown repo accepted")
163+	}
164+}
165+
166+func TestURLs(t *testing.T) {
167+	cases := map[string]string{
168+		"[email protected]:sovrn":           "[email protected]:servers",
169+		"ssh://git@host/u/sovrn":                 "ssh://git@host/u/servers",
170+		"https://host/u/sovrn.git":               "https://host/u/servers.git",
171+		"git+ssh://[email protected]/sovrn": "git+ssh://[email protected]/servers",
172+	}
173+	for of, want := range cases {
174+		if got, err := SiblingURL(of, "servers"); err != nil || got != want {
175+			t.Errorf("SiblingURL(%q) = %q, %v", of, got, err)
176+		}
177+	}
178+	if u, _ := URLFunc("git@h:{repo}", "x")("data"); u != "git@h:data" {
179+		t.Errorf("template: %s", u)
180+	}
181+	if _, err := URLFunc("", "")("data"); err == nil {
182+		t.Error("no origin and no template accepted")
183+	}
184+	if _, err := URLFunc("git@h:fixed", "")("data"); err == nil {
185+		t.Error("template without {repo} accepted")
186+	}
187+}

  1@@ -0,0 +1,144 @@
  2+package checkout
  3+
  4+import (
  5+	"context"
  6+	"fmt"
  7+	"os"
  8+	"path/filepath"
  9+	"strings"
 10+
 11+	"git.kilimanjaro.io/ci/internal/lex"
 12+)
 13+
 14+// Main says where the main checkout comes from.
 15+type Main struct {
 16+	// Repo is the repository name; the checkout is <run>/<Repo>.
 17+	Repo string
 18+	// WorkingCopy, if set, is linked as the main checkout (local runs
 19+	// without --clean): the run sees uncommitted changes.
 20+	WorkingCopy string
 21+	// Otherwise the main checkout is a clean clone of Sha from From (a local
 22+	// git dir, or a mirror) with Origin as its origin remote.
 23+	From   string
 24+	Sha    string
 25+	Origin string
 26+}
 27+
 28+// Checked is one prepared checkout.
 29+type Checked struct {
 30+	Repo string
 31+	Path string // absolute
 32+	Rev  string // commit, or "" for a linked working copy
 33+	Env  string // variable that receives Path
 34+	Link bool   // a symlink to a local directory (working copy or --repo)
 35+}
 36+
 37+// Layout prepares <runDir>/<main.Repo> and the extra repos next to it.
 38+// Overrides link an extra repo to a local directory instead of cloning it.
 39+func Layout(ctx context.Context, src *Source, runDir string, main Main, extras []lex.Repo, overrides map[string]string) ([]Checked, error) {
 40+	root := filepath.Join(runDir, main.Repo)
 41+	var out []Checked
 42+	if main.WorkingCopy != "" {
 43+		if err := link(main.WorkingCopy, root); err != nil {
 44+			return nil, err
 45+		}
 46+		out = append(out, Checked{Repo: main.Repo, Path: root, Link: true})
 47+	} else {
 48+		if err := Clone(ctx, main.From, main.Sha, root, main.Origin); err != nil {
 49+			return nil, fmt.Errorf("checkout %s: %w", main.Repo, err)
 50+		}
 51+		out = append(out, Checked{Repo: main.Repo, Path: root, Rev: main.Sha})
 52+	}
 53+
 54+	for name := range overrides {
 55+		found := false
 56+		for _, r := range extras {
 57+			found = found || r.Repo == name
 58+		}
 59+		if !found {
 60+			return nil, fmt.Errorf("--repo %s: the workflow has no repo %q", name, name)
 61+		}
 62+	}
 63+	for _, r := range extras {
 64+		dest := filepath.Clean(filepath.Join(root, r.Path))
 65+		if !Inside(runDir, dest) || dest == runDir || Inside(root, dest) {
 66+			return nil, fmt.Errorf("repo %s: path %q must be a sibling of the main checkout (like ../%s)", r.Repo, r.Path, r.Repo)
 67+		}
 68+		if p, ok := overrides[r.Repo]; ok {
 69+			if err := link(p, dest); err != nil {
 70+				return nil, err
 71+			}
 72+			out = append(out, Checked{Repo: r.Repo, Path: dest, Env: r.Env, Link: true})
 73+			continue
 74+		}
 75+		mirror, err := src.Mirror(ctx, r.Repo)
 76+		if err != nil {
 77+			return nil, err
 78+		}
 79+		sha, err := Resolve(ctx, mirror, r.Rev)
 80+		if err != nil {
 81+			return nil, fmt.Errorf("repo %s: %w", r.Repo, err)
 82+		}
 83+		url, _ := src.URL(r.Repo)
 84+		if err := Clone(ctx, mirror, sha, dest, url); err != nil {
 85+			return nil, fmt.Errorf("checkout %s: %w", r.Repo, err)
 86+		}
 87+		out = append(out, Checked{Repo: r.Repo, Path: dest, Rev: sha, Env: r.Env})
 88+	}
 89+	return out, nil
 90+}
 91+
 92+func link(target, at string) error {
 93+	abs, err := filepath.Abs(target)
 94+	if err != nil {
 95+		return err
 96+	}
 97+	if fi, err := os.Stat(abs); err != nil || !fi.IsDir() {
 98+		return fmt.Errorf("%s is not a directory", target)
 99+	}
100+	if err := os.MkdirAll(filepath.Dir(at), 0o755); err != nil {
101+		return err
102+	}
103+	return os.Symlink(abs, at)
104+}
105+
106+// SiblingURL derives a repo's clone URL from another repo's URL on the same
107+// forge by replacing the last path element: git@host:sovrn -> git@host:servers,
108+// https://host/u/sovrn.git -> https://host/u/servers.git.
109+func SiblingURL(of, repo string) (string, error) {
110+	i := strings.LastIndexAny(of, "/:")
111+	if i < 0 || i == len(of)-1 {
112+		return "", fmt.Errorf("can't derive a clone URL for %s from %q", repo, of)
113+	}
114+	suffix := ""
115+	if strings.HasSuffix(of, ".git") {
116+		suffix = ".git"
117+	}
118+	return of[:i+1] + repo + suffix, nil
119+}
120+
121+// URLFunc returns the clone-URL resolver: a template with {repo} (from
122+// CI_CLONE_URL) if set, else a sibling of origin.
123+func URLFunc(template, origin string) func(string) (string, error) {
124+	return func(repo string) (string, error) {
125+		if template != "" {
126+			if !strings.Contains(template, "{repo}") {
127+				return "", fmt.Errorf("CI_CLONE_URL %q has no {repo}", template)
128+			}
129+			return strings.ReplaceAll(template, "{repo}", repo), nil
130+		}
131+		if origin == "" {
132+			return "", fmt.Errorf("can't find a clone URL for %s: the main repo has no origin remote; set CI_CLONE_URL (e.g. git@host:{repo})", repo)
133+		}
134+		return SiblingURL(origin, repo)
135+	}
136+}
137+
138+// OriginURL returns the origin remote URL of the repository at gitDir, or "".
139+func OriginURL(ctx context.Context, gitDir string) string {
140+	out, err := git(ctx, gitDir, "config", "--get", "remote.origin.url")
141+	if err != nil {
142+		return ""
143+	}
144+	return strings.TrimSpace(out)
145+}

1@@ -33,6 +33,7 @@ const StateVersion = 1
2 type Dir string
3 
4 func (d Dir) Path() string         { return string(d) }
5+func (d Dir) Base() string         { return filepath.Base(string(d)) }
6 func (d Dir) StateFile() string    { return filepath.Join(string(d), "state.json") }
7 func (d Dir) Events() string       { return filepath.Join(string(d), "events.ndjson") }
8 func (d Dir) ManifestFile() string { return filepath.Join(string(d), "manifest.yml") }

 1@@ -66,3 +66,31 @@ func output(dir, name string, args ...string) (string, error) {
 2 	}
 3 	return string(out), nil
 4 }
 5+
 6+// Resolve returns the commit a revision names: a jj revset in jj repos
 7+// (default @, the working-copy commit), a git revision otherwise (default
 8+// HEAD). It must name exactly one commit.
 9+func Resolve(root, rev string) (string, error) {
10+	if _, err := os.Stat(filepath.Join(root, ".jj")); err == nil {
11+		if rev == "" {
12+			rev = "@"
13+		}
14+		out, err := output(root, "jj", "log", "--no-graph", "-r", rev, "-T", `commit_id ++ "\n"`)
15+		if err != nil {
16+			return "", err
17+		}
18+		ids := strings.Fields(out)
19+		if len(ids) != 1 {
20+			return "", fmt.Errorf("revision %q names %d commits, want 1", rev, len(ids))
21+		}
22+		return ids[0], nil
23+	}
24+	if rev == "" {
25+		rev = "HEAD"
26+	}
27+	out, err := output(root, "git", "rev-parse", "--verify", rev+"^{commit}")
28+	if err != nil {
29+		return "", err
30+	}
31+	return strings.TrimSpace(out), nil
32+}