From e9aa078a8572160557a4d4cfb09aa9546db952bf Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 7 Nov 2026 17:29:03 +0100 Subject: [PATCH 1/8] Mirror pkg:git tags and the submodules a commit pins A pkg:git version can now be a tag as well as a full commit ID. With a version, the submodules the commit records are mirrored at the commits it pins, from the upstream.git hosts serving them; one no host serves is skipped with a warning. --- docs/architecture.md | 2 +- docs/configuration.md | 6 +- internal/gitcache/mirror.go | 129 +++++++++++++++++++++++++--- internal/gitcache/mirror_test.go | 79 +++++++++++++++++ internal/handler/git_mirror.go | 100 ++++++++++++++++++++- internal/handler/git_mirror_test.go | 81 +++++++++++++++++ internal/handler/git_test.go | 6 +- internal/mirror/git.go | 10 +-- 8 files changed, 387 insertions(+), 26 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 655b3098..80120ee9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -363,7 +363,7 @@ Package metadata enrichment, built on [github.com/git-pkgs/enrichment](https://g ### `internal/mirror` -Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and commit in the git cache that `/git` serves. +Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and the commit or tag, with its submodules, in the git cache that `/git` serves. The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints. diff --git a/docs/configuration.md b/docs/configuration.md index 584896bc..1c015890 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1910,16 +1910,18 @@ proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403 ### Git repositories -`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID: +`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID or a tag: ```bash proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567 +proxy mirror pkg:git/github.com/git-pkgs/purl@v0.2.1 proxy mirror pkg:git/github.com/git-pkgs/purl ``` - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. - The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. -- Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID. +- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name as the version fails too; give the commit ID. +- With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. ### Scheduled refresh diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index b5bde40f..047934fb 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -4,27 +4,44 @@ import ( "context" "errors" "fmt" + "os/exec" + "slices" "strings" "time" ) // MirrorResult is what Mirror did. type MirrorResult struct { - // Cached reports that the repository, and the commit if one was given, - // were cached already. + // Cached reports that the repository, and the commit if a version was + // given, were cached already. Cached bool // Bytes is how much the cached repository grew. Bytes int64 + // Commit is the commit the version names, empty without a version. + Commit string + // Submodules are the submodules Commit records, sorted by path. + Submodules []Submodule +} + +// Submodule is a submodule a commit records: its path, the URL .gitmodules +// gives for it, which may be relative to the repository's, and the commit +// the gitlink pins. +type Submodule struct { + Path, URL, Commit string } // Mirror brings the repository cached under key up to date with repoURL, -// cloning it if needed, and waits for that to finish. With commit, an object -// ID, it also makes sure the commit is cached. One on no branch or tag is -// fetched by ID when fetchWants allows it (see OpenOptions.FetchWants), and -// is kept as long as a commit a client fetched by ID. -func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWants bool) (MirrorResult, error) { - if commit != "" && !isObjectID(commit) { - return MirrorResult{}, fmt.Errorf("%q is not a full commit ID", commit) +// cloning it if needed, and waits for that to finish. version, if given, is +// a full commit ID or a tag. A commit is made sure to be cached: one on no +// branch or tag is fetched by ID when fetchWants allows it (see +// OpenOptions.FetchWants), and is kept as long as a commit a client fetched +// by ID. A tag is cached with the repository's tags. +func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchWants bool) (MirrorResult, error) { + tag := version != "" && !isObjectID(version) + if tag { + if err := c.checkTagName(ctx, version); err != nil { + return MirrorResult{}, err + } } lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants}) if err != nil { @@ -53,17 +70,105 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWa if errors.Is(refreshErr, ErrTooLarge) { return MirrorResult{}, refreshErr } - if commit != "" { + commit := version + switch { + case tag: + if commit, err = c.resolveTag(ctx, rp.dir, version); err != nil { + if refreshErr != nil { + // The tag may be one the failed refresh would have brought. + return MirrorResult{}, refreshErr + } + return MirrorResult{}, err + } + case version != "": if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil { return MirrorResult{}, err } } // A refresh that failed leaves the commit cached all the same. - if refreshErr != nil && commit == "" { + if refreshErr != nil && version == "" { return MirrorResult{}, refreshErr } + var submodules []Submodule + if commit != "" { + if submodules, err = c.submodules(ctx, rp.dir, commit); err != nil { + return MirrorResult{}, err + } + } grown := max(rp.size.Load()-before, 0) - return MirrorResult{Cached: cached && grown == 0, Bytes: grown}, nil + return MirrorResult{Cached: cached && grown == 0, Bytes: grown, Commit: commit, Submodules: submodules}, nil +} + +// checkTagName refuses a version that is no valid tag name, such as one +// with .. or a revision suffix like ^{tree}. +func (c *Cache) checkTagName(ctx context.Context, name string) error { + if err := c.command(ctx, nil, "check-ref-format", "refs/tags/"+name).Run(); err != nil { + return fmt.Errorf("%q is neither a full commit ID nor a tag name", name) + } + return nil +} + +// resolveTag returns the commit tag points at. +func (c *Cache) resolveTag(ctx context.Context, dir, tag string) (string, error) { + out, err := c.command(ctx, nil, "-C", dir, "rev-parse", "--verify", "--quiet", "--end-of-options", "refs/tags/"+tag+"^{commit}").Output() + var exit *exec.ExitError + if errors.As(err, &exit) && exit.ExitCode() == 1 { + return "", fmt.Errorf("tag %s: %w", tag, ErrNotFound) + } + if err != nil { + return "", fmt.Errorf("resolving tag %s: %w", tag, err) + } + return strings.TrimSpace(string(out)), nil +} + +// submodules lists the submodules commit records: its gitlinks that +// .gitmodules names a URL for. +func (c *Cache) submodules(ctx context.Context, dir, commit string) ([]Submodule, error) { + if c.command(ctx, nil, "-C", dir, "cat-file", "-e", commit+":.gitmodules").Run() != nil { + return nil, nil + } + out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--get-regexp", `^submodule\..*\.(path|url)$`) + if err != nil { + return nil, fmt.Errorf("reading .gitmodules: %w", err) + } + paths, urls := map[string]string{}, map[string]string{} + for entry := range strings.SplitSeq(out, "\x00") { + key, value, ok := strings.Cut(entry, "\n") + name, field, dotted := cutLast(strings.TrimPrefix(key, "submodule."), ".") + if !ok || !dotted { + continue + } + if field == "path" { + paths[value] = name + } else { + urls[name] = value + } + } + tree, err := c.run(ctx, dir, "ls-tree", "-r", "-z", "--full-tree", commit) + if err != nil { + return nil, err + } + var submodules []Submodule + for entry := range strings.SplitSeq(tree, "\x00") { + meta, path, ok := strings.Cut(entry, "\t") + fields := strings.Fields(meta) + if !ok || len(fields) != 3 || fields[0] != "160000" { + continue + } + if url := urls[paths[path]]; url != "" && isObjectID(fields[2]) { + submodules = append(submodules, Submodule{Path: path, URL: url, Commit: fields[2]}) + } + } + slices.SortFunc(submodules, func(a, b Submodule) int { return strings.Compare(a.Path, b.Path) }) + return submodules, nil +} + +func cutLast(s, sep string) (before, after string, found bool) { + i := strings.LastIndex(s, sep) + if i < 0 { + return s, "", false + } + return s[:i], s[i+len(sep):], true } // ensureCommit makes sure the repository has commit, fetching it by ID when diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index 86f652f9..499acfb3 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -212,3 +212,82 @@ func TestMirrorReportsRefreshTooLarge(t *testing.T) { t.Errorf("mirror with a refresh past max_repo_size = %v, want ErrTooLarge", err) } } + +func TestMirrorResolvesTag(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tagged := runGit(t, work, "rev-parse", "v1^{commit}") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Commit != tagged { + t.Errorf("commit %q, want %s the tag points at", got.Commit, tagged) + } +} + +func TestMirrorReportsMissingTag(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, version := range []string{"v9", "main"} { + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", version, true) + if !errors.Is(err, ErrNotFound) { + t.Errorf("mirror at %q = %v, want ErrNotFound", version, err) + } + } +} + +func TestMirrorRefusesInvalidTag(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, version := range []string{"a..b", "v1^{tree}", "v1~1", "v 1", "v1:x"} { + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", version, true) + if err == nil || errors.Is(err, ErrNotFound) { + t.Errorf("mirror at %q = %v, want an invalid version error", version, err) + } + } +} + +func TestMirrorListsSubmodules(t *testing.T) { + upstream := newFakeUpstream(t) + libWork := upstream.newRepo("lib") + libCommit := runGit(t, libWork, "rev-parse", "HEAD") + work := upstream.newRepo("repo") + gitmodules := "[submodule \"vendor/lib\"]\n\tpath = vendor/lib\n\turl = ../lib.git\n" + + "[submodule \"other\"]\n\tpath = third_party/other\n\turl = https://example.com/other.git\n" + if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(gitmodules), 0o600); err != nil { + t.Fatal(err) + } + otherCommit := strings.Repeat("ab", 20) + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",vendor/lib") + runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+otherCommit+",third_party/other") + runGit(t, work, "commit", "--quiet", "-m", "submodules") + runGit(t, work, "push", "--quiet", "origin", "main") + commit := runGit(t, work, "rev-parse", "HEAD") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + want := []Submodule{ + {Path: "third_party/other", URL: "https://example.com/other.git", Commit: otherCommit}, + {Path: "vendor/lib", URL: "../lib.git", Commit: libCommit}, + } + if !slices.Equal(got.Submodules, want) { + t.Errorf("submodules %+v, want %+v", got.Submodules, want) + } + + // Without a version, nothing names a commit to read submodules from. + got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil || len(got.Submodules) != 0 { + t.Errorf("unversioned mirror = %+v, %v, want no submodules", got, err) + } +} diff --git a/internal/handler/git_mirror.go b/internal/handler/git_mirror.go index 709df313..38be233c 100644 --- a/internal/handler/git_mirror.go +++ b/internal/handler/git_mirror.go @@ -3,22 +3,114 @@ package handler import ( "context" "fmt" + "log/slog" "net/url" + "path" "strings" "github.com/git-pkgs/proxy/internal/gitcache" ) +// maxSubmoduleDepth bounds how deeply Mirror follows submodules of +// submodules. +const maxSubmoduleDepth = 8 + // Mirror caches the repository at location, a host and path such as // github.com/acme/widgets as a pkg:git PURL names it, from the upstream.git -// host serving it, and with commit, a full commit ID, that commit too. -// Clones through /git are then served from that copy. -func (h *GitHandler) Mirror(ctx context.Context, location, commit string) (gitcache.MirrorResult, error) { +// host serving it. With version, a full commit ID or a tag, it caches that +// commit too, and the commits its submodules pin, from the upstream.git +// hosts serving them. A submodule no upstream.git host serves is skipped. +// Clones through /git are then served from those copies. +func (h *GitHandler) Mirror(ctx context.Context, location, version string) (gitcache.MirrorResult, error) { + return h.mirror(ctx, location, version, map[string]bool{}, 0) +} + +func (h *GitHandler) mirror(ctx context.Context, location, version string, seen map[string]bool, depth int) (gitcache.MirrorResult, error) { repo, err := h.mirrorRepository(location) if err != nil { return gitcache.MirrorResult{}, err } - return h.cache.Mirror(ctx, repo.key, repo.url, commit, repo.upstream.fetchWants) + result, err := h.cache.Mirror(ctx, repo.key, repo.url, version, repo.upstream.fetchWants) + if err != nil { + return result, err + } + for _, sub := range result.Submodules { + subLocation, err := gitSubmoduleLocation(repo.url, sub.URL) + if err == nil { + _, err = h.mirrorRepository(subLocation) + } + if err != nil { + h.logger().Warn("git mirror: skipping submodule", "repository", location, "submodule", sub.Path, "url", sub.URL, "error", err) + continue + } + if seen[subLocation+"@"+sub.Commit] { + continue + } + seen[subLocation+"@"+sub.Commit] = true + if depth >= maxSubmoduleDepth { + return result, fmt.Errorf("submodule %s: nested more than %d deep", sub.Path, maxSubmoduleDepth) + } + subResult, err := h.mirror(ctx, subLocation, sub.Commit, seen, depth+1) + if err != nil { + return result, fmt.Errorf("submodule %s: %w", sub.Path, err) + } + result.Cached = result.Cached && subResult.Cached + result.Bytes += subResult.Bytes + } + return result, nil +} + +func (h *GitHandler) logger() *slog.Logger { + if h.proxy != nil && h.proxy.Logger != nil { + return h.proxy.Logger + } + return slog.Default() +} + +// gitSubmoduleLocation turns the URL .gitmodules gives a submodule into a +// location as Mirror takes it. A relative URL is resolved against the +// superproject's repoURL, as git does; ssh, git and scp-like URLs name the +// host and path the repository has there. +func gitSubmoduleLocation(repoURL, submoduleURL string) (string, error) { + var host, repoPath string + switch { + case strings.HasPrefix(submoduleURL, "./"), strings.HasPrefix(submoduleURL, "../"): + u, err := url.Parse(repoURL) + if err != nil { + return "", err + } + host, repoPath = u.Host, path.Join(u.Path, submoduleURL) + case strings.Contains(submoduleURL, "://"): + u, err := url.Parse(submoduleURL) + if err != nil { + return "", err + } + switch u.Scheme { + case "http", "https": + host = u.Host + case "ssh", "git", "git+ssh", "ssh+git": + host = u.Hostname() + default: + return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) + } + repoPath = u.Path + default: + // scp-like: [user@]host:path + hostPart, p, ok := strings.Cut(submoduleURL, ":") + if !ok || strings.Contains(hostPart, "/") { + return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) + } + _, host, _ = strings.Cut(hostPart, "@") + if host == "" { + host = hostPart + } + repoPath = p + } + repoPath = strings.Trim(path.Clean("/"+repoPath), "/") + if host == "" || repoPath == "" { + return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) + } + return strings.ToLower(host) + "/" + repoPath, nil } // mirrorRepository resolves location to a repository of the upstream.git diff --git a/internal/handler/git_mirror_test.go b/internal/handler/git_mirror_test.go index 4ebcf982..227bd71e 100644 --- a/internal/handler/git_mirror_test.go +++ b/internal/handler/git_mirror_test.go @@ -2,6 +2,8 @@ package handler import ( "context" + "os" + "path/filepath" "strings" "testing" @@ -71,3 +73,82 @@ func TestGitMirrorCachesCommitForClones(t *testing.T) { t.Errorf("cloned HEAD %s, want %s", head, sha) } } + +func TestGitSubmoduleLocation(t *testing.T) { + tests := []struct { + repoURL, submoduleURL, want string + }{ + {"https://github.com/acme/widgets", "../lib.git", "github.com/acme/lib.git"}, + {"https://github.com/acme/widgets.git", "../../other/lib", "github.com/other/lib"}, + {"https://github.com/acme/widgets", "./vendored", "github.com/acme/widgets/vendored"}, + {"http://127.0.0.1:8080/team/repo", "../lib.git", "127.0.0.1:8080/team/lib.git"}, + {"https://github.com/acme/widgets", "https://GitLab.com/group/lib.git/", "gitlab.com/group/lib.git"}, + {"https://github.com/acme/widgets", "ssh://git@github.com:22/acme/lib.git", "github.com/acme/lib.git"}, + {"https://github.com/acme/widgets", "git://git.example/lib", "git.example/lib"}, + {"https://github.com/acme/widgets", "git@github.com:acme/lib.git", "github.com/acme/lib.git"}, + {"https://github.com/acme/widgets", "/srv/git/lib.git", ""}, + {"https://github.com/acme/widgets", "file:///srv/git/lib.git", ""}, + {"https://github.com/acme/widgets", "", ""}, + } + for _, tt := range tests { + got, err := gitSubmoduleLocation(tt.repoURL, tt.submoduleURL) + if tt.want == "" { + if err == nil { + t.Errorf("location of %q = %q, want an error", tt.submoduleURL, got) + } + continue + } + if err != nil || got != tt.want { + t.Errorf("location of %q from %s = %q, %v; want %q", tt.submoduleURL, tt.repoURL, got, err, tt.want) + } + } +} + +// TestGitMirrorCachesSubmodules mirrors a commit with a submodule served by +// the same upstream and one on a host no upstream.git serves, which is +// skipped. +func TestGitMirrorCachesSubmodules(t *testing.T) { + g := newGitTestSetup(t, 0, 0) + libBare := filepath.Join(g.root, "lib.git") + runGitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", libBare) + libWork := t.TempDir() + runGitCommand(t, "", "clone", "--quiet", libBare, libWork) + runGitCommand(t, libWork, "commit", "--quiet", "--allow-empty", "-m", "lib") + runGitCommand(t, libWork, "push", "--quiet", "origin", "HEAD:main") + libCommit := runGitCommand(t, libWork, "rev-parse", "HEAD") + + gitmodules := "[submodule \"lib\"]\n\tpath = lib\n\turl = ../lib.git\n" + + "[submodule \"elsewhere\"]\n\tpath = elsewhere\n\turl = https://elsewhere.example/x.git\n" + if err := os.WriteFile(filepath.Join(g.work, ".gitmodules"), []byte(gitmodules), 0o600); err != nil { + t.Fatal(err) + } + runGitCommand(t, g.work, "add", ".gitmodules") + runGitCommand(t, g.work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",lib") + runGitCommand(t, g.work, "update-index", "--add", "--cacheinfo", "160000,"+strings.Repeat("ab", 20)+",elsewhere") + runGitCommand(t, g.work, "commit", "--quiet", "-m", "submodules") + runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") + sha := runGitCommand(t, g.work, "rev-parse", "HEAD") + location := strings.TrimPrefix(g.upstreamURL, "http://") + "/repo" + + got, err := g.handler.Mirror(context.Background(), location, sha) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached { + t.Errorf("first mirror = %+v, want fetched", got) + } + if got, err = g.handler.Mirror(context.Background(), location, sha); err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } + if !strings.Contains(g.logs.String(), "elsewhere.example") { + t.Errorf("log doesn't name the skipped submodule:\n%s", g.logs.String()) + } + + // With the upstream gone, the submodule's commit is served from the cache. + g.upstream.Close() + clone := t.TempDir() + runGitCommand(t, "", "clone", "--quiet", g.server.URL+"/git/up/lib", clone) + if head := runGitCommand(t, clone, "rev-parse", "HEAD"); head != libCommit { + t.Errorf("cloned lib HEAD %s, want %s", head, libCommit) + } +} diff --git a/internal/handler/git_test.go b/internal/handler/git_test.go index cf553d60..aeec2bc9 100644 --- a/internal/handler/git_test.go +++ b/internal/handler/git_test.go @@ -129,7 +129,9 @@ func runGitCommand(t *testing.T, dir string, args ...string) string { // gitTestSetup is a GitHandler in front of a cache of one upstream // repository, "repo", served by git http-backend. type gitTestSetup struct { - work string + work string + // root holds the upstream's bare repositories. + root string server *httptest.Server // upstream serves the repository, at upstreamURL/repo, and handler is // the GitHandler under test. @@ -161,7 +163,7 @@ func newGitTestSetup(t *testing.T, readTimeout, writeStall time.Duration) *gitTe bare := filepath.Join(root, "repo.git") runGitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) runGitCommand(t, bare, "config", "uploadpack.allowAnySHA1InWant", "true") - g := &gitTestSetup{work: t.TempDir(), logs: &syncLog{}} + g := &gitTestSetup{work: t.TempDir(), root: root, logs: &syncLog{}} runGitCommand(t, "", "clone", "--quiet", bare, g.work) runGitCommand(t, g.work, "commit", "--quiet", "--allow-empty", "-m", "first") runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") diff --git a/internal/mirror/git.go b/internal/mirror/git.go index a4908dc5..e9c04ffe 100644 --- a/internal/mirror/git.go +++ b/internal/mirror/git.go @@ -9,10 +9,10 @@ import ( // GitMirrorer caches Git repositories for pkg:git items. location is the // repository's host and path, such as codeberg.org/forgejo/forgejo, and -// commit a full commit ID, or empty to cache the repository's branches and -// tags only. +// version a full commit ID or a tag, or empty to cache the repository's +// branches and tags only. type GitMirrorer interface { - Mirror(ctx context.Context, location, commit string) (gitcache.MirrorResult, error) + Mirror(ctx context.Context, location, version string) (gitcache.MirrorResult, error) } // UnavailableGit fails every pkg:git item with Reason. @@ -26,8 +26,8 @@ func (u UnavailableGit) Mirror(context.Context, string, string) (gitcache.Mirror var errNoGitUpstream = errors.New("git repositories are mirrored from upstream.git, and none is configured") -// mirrorGit caches a pkg:git repository, and the commit its version names, -// in the cache /git serves clones from. +// mirrorGit caches a pkg:git repository, and the commit its version names +// with the submodules it pins, in the cache /git serves clones from. func (m *Mirror) mirrorGit(ctx context.Context, pv PackageVersion, tracker *progressTracker) { if m.Git == nil { m.recordFailure(tracker, pv, errNoGitUpstream) From 7f2ac614c421a5b21e6eba71e82fb570e47c9dca Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 7 Nov 2026 17:51:13 +0100 Subject: [PATCH 2/8] Mirror commits whose .gitmodules lists no submodule git config --get-regexp exits 1 when nothing matches, which failed the mirror of a commit with an empty .gitmodules. The submodules are now read from --list. --- internal/gitcache/mirror.go | 13 ++++++++----- internal/gitcache/mirror_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index 047934fb..5aaa6fba 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -127,20 +127,23 @@ func (c *Cache) submodules(ctx context.Context, dir, commit string) ([]Submodule if c.command(ctx, nil, "-C", dir, "cat-file", "-e", commit+":.gitmodules").Run() != nil { return nil, nil } - out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--get-regexp", `^submodule\..*\.(path|url)$`) + // --list rather than --get-regexp, which fails when nothing matches. + out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--list") if err != nil { return nil, fmt.Errorf("reading .gitmodules: %w", err) } paths, urls := map[string]string{}, map[string]string{} for entry := range strings.SplitSeq(out, "\x00") { key, value, ok := strings.Cut(entry, "\n") - name, field, dotted := cutLast(strings.TrimPrefix(key, "submodule."), ".") - if !ok || !dotted { + section, ok2 := strings.CutPrefix(key, "submodule.") + name, field, dotted := cutLast(section, ".") + if !ok || !ok2 || !dotted { continue } - if field == "path" { + switch field { + case "path": paths[value] = name - } else { + case "url": urls[name] = value } } diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index 499acfb3..1f85a6cc 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -291,3 +291,27 @@ func TestMirrorListsSubmodules(t *testing.T) { t.Errorf("unversioned mirror = %+v, %v, want no submodules", got, err) } } + +// TestMirrorWithEmptyGitmodules checks a .gitmodules with no path or url, +// for which git config --get-regexp finds nothing, lists no submodules. +func TestMirrorWithEmptyGitmodules(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + for i, content := range []string{"", "[submodule \"x\"]\n\tbranch = main\n"} { + if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "commit", "--quiet", "--allow-empty", "-m", "gitmodules "+strconv.Itoa(i)) + } + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, rev := range []string{"HEAD~1", "HEAD"} { + commit := runGit(t, work, "rev-parse", rev) + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) + if err != nil || len(got.Submodules) != 0 { + t.Errorf("mirror of %s = %+v, %v, want no submodules", rev, got, err) + } + } +} From 7de58c523f21307ab829d6b2cd0606ecb0d9af3d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:08:40 +0100 Subject: [PATCH 3/8] Report errors reading submodules of a mirrored commit Looking for .gitmodules took any error from git cat-file as the file not being there, so a mirror whose context was cancelled reported success with no submodules. Use rev-parse --verify --quiet, whose exit status 1 means the path is missing, and return other errors. A .gitmodules git config can't parse failed the mirror of a commit that is cached all the same. Log a warning and skip its submodules instead, as an unserved submodule host is skipped. --- docs/configuration.md | 2 +- internal/gitcache/helpers_test.go | 3 +- internal/gitcache/mirror.go | 22 +++++++++++--- internal/gitcache/mirror_test.go | 50 +++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 1c015890..a1ced4b4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1921,7 +1921,7 @@ proxy mirror pkg:git/github.com/git-pkgs/purl - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. - The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. - Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name as the version fails too; give the commit ID. -- With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log. +- With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log, as are all of a commit's submodules when git can't read its `.gitmodules`. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. ### Scheduled refresh diff --git a/internal/gitcache/helpers_test.go b/internal/gitcache/helpers_test.go index 55b73335..1f18c5b2 100644 --- a/internal/gitcache/helpers_test.go +++ b/internal/gitcache/helpers_test.go @@ -302,6 +302,7 @@ func (u *fakeUpstream) bare() string { type testCache struct { *Cache server *httptest.Server + logs *lockedBuffer } func newTestCache(t *testing.T, upstreamURL string, configure func(*Options)) *testCache { @@ -333,7 +334,7 @@ func newTestCache(t *testing.T, upstreamURL string, configure func(*Options)) *t server.Close() _ = c.Close() }) - return &testCache{Cache: c, server: server} + return &testCache{Cache: c, server: server, logs: logs} } // testFront serves /git/up/{repo}/info/refs and git-upload-pack from c, as diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index 5aaa6fba..d6e07f9e 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -91,7 +91,7 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchW } var submodules []Submodule if commit != "" { - if submodules, err = c.submodules(ctx, rp.dir, commit); err != nil { + if submodules, err = c.submodules(ctx, key, rp.dir, commit); err != nil { return MirrorResult{}, err } } @@ -123,14 +123,28 @@ func (c *Cache) resolveTag(ctx context.Context, dir, tag string) (string, error) // submodules lists the submodules commit records: its gitlinks that // .gitmodules names a URL for. -func (c *Cache) submodules(ctx context.Context, dir, commit string) ([]Submodule, error) { - if c.command(ctx, nil, "-C", dir, "cat-file", "-e", commit+":.gitmodules").Run() != nil { +func (c *Cache) submodules(ctx context.Context, key, dir, commit string) ([]Submodule, error) { + err := c.command(ctx, nil, "-C", dir, "rev-parse", "--verify", "--quiet", "--end-of-options", commit+":.gitmodules").Run() + var exit *exec.ExitError + if errors.As(err, &exit) && exit.ExitCode() == 1 && ctx.Err() == nil { return nil, nil } + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf("looking for .gitmodules: %w", err) + } // --list rather than --get-regexp, which fails when nothing matches. out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--list") if err != nil { - return nil, fmt.Errorf("reading .gitmodules: %w", err) + if ctx.Err() != nil { + return nil, ctx.Err() + } + // The commit is cached; a .gitmodules git can't read only loses + // its submodules. + c.logger.Warn("git cache: cannot read .gitmodules, skipping submodules", "repository", key, "commit", commit, "error", err) + return nil, nil } paths, urls := map[string]string{}, map[string]string{} for entry := range strings.SplitSeq(out, "\x00") { diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index 1f85a6cc..e57e64f0 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -315,3 +315,53 @@ func TestMirrorWithEmptyGitmodules(t *testing.T) { } } } + +// TestSubmodulesReportsCancel checks a cancelled context is reported, not +// read as a commit without .gitmodules. +func TestSubmodulesReportsCancel(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + commit := runGit(t, work, "rev-parse", "HEAD") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + tc := newTestCache(t, upstream.server.URL, nil) + + if _, err := tc.submodules(ctx, "up/repo.git", work, commit); !errors.Is(err, context.Canceled) { + t.Errorf("submodules with a cancelled context = %v, want context.Canceled", err) + } +} + +// TestMirrorSkipsBrokenGitmodules checks a .gitmodules git can't read, one +// with a syntax error or committed as a symlink, lists no submodules and +// logs a warning instead of failing the mirror. +func TestMirrorSkipsBrokenGitmodules(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + gitmodules := filepath.Join(work, ".gitmodules") + if err := os.WriteFile(gitmodules, []byte("[submodule \"x\"\n\tpath = x\n"), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "commit", "--quiet", "-m", "broken gitmodules") + if err := os.Remove(gitmodules); err != nil { + t.Fatal(err) + } + if err := os.Symlink("../outside/.gitmodules", gitmodules); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "commit", "--quiet", "-m", "symlinked gitmodules") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, rev := range []string{"HEAD~1", "HEAD"} { + commit := runGit(t, work, "rev-parse", rev) + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) + if err != nil || len(got.Submodules) != 0 { + t.Errorf("mirror of %s = %+v, %v, want no submodules", rev, got, err) + } + if !strings.Contains(tc.logs.String(), "commit="+commit) { + t.Errorf("no warning naming %s in the log", commit) + } + } +} From 989697596f9f4fa07e80aa74c198094d25e8bbba Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:09:25 +0100 Subject: [PATCH 4/8] Accept uppercase commit IDs and explain short ones when mirroring A version that wasn't a lowercase full-length hex ID was taken for a tag, so an uppercase commit ID failed as a missing tag. Lowercase a full-length hex version first. A shorter hex version that names no tag now fails saying it is neither a full commit ID nor a tag. --- docs/configuration.md | 2 +- internal/gitcache/mirror.go | 7 +++++++ internal/gitcache/mirror_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index a1ced4b4..0d816460 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1920,7 +1920,7 @@ proxy mirror pkg:git/github.com/git-pkgs/purl - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. - The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. -- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name as the version fails too; give the commit ID. +- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name or an abbreviated commit ID as the version fails too; give the full commit ID. - With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log, as are all of a commit's submodules when git can't read its `.gitmodules`. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index d6e07f9e..f884d37f 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -37,6 +37,9 @@ type Submodule struct { // OpenOptions.FetchWants), and is kept as long as a commit a client fetched // by ID. A tag is cached with the repository's tags. func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchWants bool) (MirrorResult, error) { + if lower := strings.ToLower(version); isObjectID(lower) { + version = lower + } tag := version != "" && !isObjectID(version) if tag { if err := c.checkTagName(ctx, version); err != nil { @@ -78,6 +81,10 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchW // The tag may be one the failed refresh would have brought. return MirrorResult{}, refreshErr } + if errors.Is(err, ErrNotFound) && strings.Trim(strings.ToLower(version), "0123456789abcdef") == "" { + // Likely an abbreviated commit ID. + return MirrorResult{}, fmt.Errorf("%s is not a full commit ID or tag: %w", version, ErrNotFound) + } return MirrorResult{}, err } case version != "": diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index e57e64f0..b1e27f61 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -365,3 +365,27 @@ func TestMirrorSkipsBrokenGitmodules(t *testing.T) { } } } + +func TestMirrorAcceptsUppercaseCommit(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + commit := runGit(t, work, "rev-parse", "HEAD") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", strings.ToUpper(commit), true) + if err != nil || got.Commit != commit { + t.Errorf("mirror of the uppercase commit ID = %+v, %v, want commit %s", got, err, commit) + } +} + +func TestMirrorExplainsShortCommit(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + commit := runGit(t, work, "rev-parse", "HEAD") + tc := newTestCache(t, upstream.server.URL, nil) + + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit[:12], true) + if !errors.Is(err, ErrNotFound) || !strings.Contains(err.Error(), "full commit ID") { + t.Errorf("mirror of a short commit ID = %v, want ErrNotFound saying it is no full commit ID", err) + } +} From 6d80291d22334e14fb928d263c7848b56499e402 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:11:36 +0100 Subject: [PATCH 5/8] Warn when mirroring a cached tag after a failed refresh A tag found in the cache after the refresh failed was reported as mirrored with no sign it might be stale, though tags can move. Keep serving it, as a cached commit is, but log a warning naming the tag, the commit and the refresh error. --- docs/configuration.md | 2 +- internal/gitcache/mirror.go | 4 ++++ internal/gitcache/mirror_test.go | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0d816460..59e59605 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1920,7 +1920,7 @@ proxy mirror pkg:git/github.com/git-pkgs/purl - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. - The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. -- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name or an abbreviated commit ID as the version fails too; give the full commit ID. +- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. When the upstream can't be reached, a cached tag is mirrored at the commit it last pointed to, with a warning in the log. A branch name or an abbreviated commit ID as the version fails too; give the full commit ID. - With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log, as are all of a commit's submodules when git can't read its `.gitmodules`. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index f884d37f..ffcd0d9c 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -87,6 +87,10 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchW } return MirrorResult{}, err } + if refreshErr != nil { + // Tags can move; this is where it pointed when last fetched. + c.logger.Warn("git cache: refresh failed, mirroring the cached tag", "repository", key, "tag", version, "commit", commit, "error", refreshErr) + } case version != "": if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil { return MirrorResult{}, err diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index b1e27f61..17a19f07 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -3,6 +3,7 @@ package gitcache import ( "context" "errors" + "net/http" "os" "path/filepath" "slices" @@ -389,3 +390,22 @@ func TestMirrorExplainsShortCommit(t *testing.T) { t.Errorf("mirror of a short commit ID = %v, want ErrNotFound saying it is no full commit ID", err) } } + +// TestMirrorWarnsOfStaleTag checks a tag served from the cache when the +// refresh fails is mirrored with a warning, since the tag may have moved. +func TestMirrorWarnsOfStaleTag(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true); err != nil { + t.Fatalf("Mirror: %v", err) + } + upstream.status.Store(http.StatusServiceUnavailable) + + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true); err != nil { + t.Fatalf("mirror of a cached tag with the upstream down: %v", err) + } + if !strings.Contains(tc.logs.String(), "tag=v1") { + t.Errorf("no warning naming the tag in the log:\n%s", tc.logs.String()) + } +} From 0829eae376be23dedc391d2f73f4eec2965b16d3 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 8 Nov 2026 17:19:03 +0000 Subject: [PATCH 6/8] List only the gitlinks .gitmodules names ls-tree -r read every path in the commit to find a few submodules. --- internal/gitcache/mirror.go | 12 +++++++++++- internal/gitcache/mirror_test.go | 5 ++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index ffcd0d9c..0d76a46e 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -172,7 +172,17 @@ func (c *Cache) submodules(ctx context.Context, key, dir, commit string) ([]Subm urls[name] = value } } - tree, err := c.run(ctx, dir, "ls-tree", "-r", "-z", "--full-tree", commit) + var pathspecs []string + for path, name := range paths { + if path != "" && urls[name] != "" { + pathspecs = append(pathspecs, path) + } + } + if len(pathspecs) == 0 { + return nil, nil + } + args := append([]string{"--literal-pathspecs", "ls-tree", "-z", "--full-tree", commit, "--"}, pathspecs...) + tree, err := c.run(ctx, dir, args...) if err != nil { return nil, err } diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index 17a19f07..ca734f11 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -261,7 +261,8 @@ func TestMirrorListsSubmodules(t *testing.T) { libCommit := runGit(t, libWork, "rev-parse", "HEAD") work := upstream.newRepo("repo") gitmodules := "[submodule \"vendor/lib\"]\n\tpath = vendor/lib\n\turl = ../lib.git\n" + - "[submodule \"other\"]\n\tpath = third_party/other\n\turl = https://example.com/other.git\n" + "[submodule \"other\"]\n\tpath = third_party/other\n\turl = https://example.com/other.git\n" + + "[submodule \"colon\"]\n\tpath = :colon\n\turl = ../lib.git\n" if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(gitmodules), 0o600); err != nil { t.Fatal(err) } @@ -269,6 +270,7 @@ func TestMirrorListsSubmodules(t *testing.T) { runGit(t, work, "add", ".gitmodules") runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",vendor/lib") runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+otherCommit+",third_party/other") + runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",:colon") runGit(t, work, "commit", "--quiet", "-m", "submodules") runGit(t, work, "push", "--quiet", "origin", "main") commit := runGit(t, work, "rev-parse", "HEAD") @@ -279,6 +281,7 @@ func TestMirrorListsSubmodules(t *testing.T) { t.Fatalf("Mirror: %v", err) } want := []Submodule{ + {Path: ":colon", URL: "../lib.git", Commit: libCommit}, {Path: "third_party/other", URL: "https://example.com/other.git", Commit: otherCommit}, {Path: "vendor/lib", URL: "../lib.git", Commit: libCommit}, } From a330d68564b6034aef3307ea9114e4eff412b7e8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 18:58:40 +0100 Subject: [PATCH 7/8] Use strings.CutLast for .gitmodules keys Go 1.27 is on main now. --- internal/gitcache/mirror.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index c59e9053..6e45d439 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -161,7 +161,7 @@ func (c *Cache) submodules(ctx context.Context, key, dir, commit string) ([]Subm for entry := range strings.SplitSeq(out, "\x00") { key, value, ok := strings.Cut(entry, "\n") section, ok2 := strings.CutPrefix(key, "submodule.") - name, field, dotted := cutLast(section, ".") + name, field, dotted := strings.CutLast(section, ".") if !ok || !ok2 || !dotted { continue } @@ -201,14 +201,6 @@ func (c *Cache) submodules(ctx context.Context, key, dir, commit string) ([]Subm return submodules, nil } -func cutLast(s, sep string) (before, after string, found bool) { - i := strings.LastIndex(s, sep) - if i < 0 { - return s, "", false - } - return s[:i], s[i+len(sep):], true -} - // ensureCommit makes sure the repository has commit, fetching it by ID when // it doesn't and fetchWants allows it. A commit on no branch or tag that is // cached already gets a new wanted ref, so it is kept for wantedMaxAge from From 0efe54fd73ff3c2aaf30372db48ee715b25d07f2 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 19:11:36 +0100 Subject: [PATCH 8/8] Let the submodule test add a ':' path on Windows Git for Windows refuses it with core.protectNTFS on. --- internal/gitcache/mirror_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index ca734f11..d016f2a1 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -270,7 +270,8 @@ func TestMirrorListsSubmodules(t *testing.T) { runGit(t, work, "add", ".gitmodules") runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",vendor/lib") runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+otherCommit+",third_party/other") - runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",:colon") + // Git for Windows refuses ':' in paths unless NTFS protection is off. + runGit(t, work, "-c", "core.protectNTFS=false", "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",:colon") runGit(t, work, "commit", "--quiet", "-m", "submodules") runGit(t, work, "push", "--quiet", "origin", "main") commit := runGit(t, work, "rev-parse", "HEAD")