Merge pull request #674 from simonchrz/feat/mirror-git-tags-submodules

Mirror pkg:git tags and submodules

Author: andrew

docs/architecture.md

@@ -368,7 +368,7 @@ Package metadata enrichment, built on [github.com/git-pkgs/enrichment](https://g
 
 ### `internal/mirror`
 
-Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and commit in the git cache that `/git` serves.
+Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and the commit or tag, with its submodules, in the git cache that `/git` serves.
 
 The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints.
 

docs/configuration.md

@@ -1927,16 +1927,18 @@ proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403
 
 ### Git repositories
 
-`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID:
+`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID or a tag:
 
 ```bash
 proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567
+proxy mirror pkg:git/github.com/git-pkgs/purl@v0.2.1
 proxy mirror pkg:git/github.com/git-pkgs/purl
 ```
 
 - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail.
 - The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails.
-- Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID.
+- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. When the upstream can't be reached, a cached tag is mirrored at the commit it last pointed to, with a warning in the log. A branch name or an abbreviated commit ID as the version fails too; give the full commit ID.
+- With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log, as are all of a commit's submodules when git can't read its `.gitmodules`.
 - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead.
 
 ### Scheduled refresh

internal/gitcache/helpers_test.go

@@ -316,6 +316,7 @@ func (u *fakeUpstream) bare() string {
 type testCache struct {
 	*Cache
 	server *httptest.Server
+	logs   *lockedBuffer
 }
 
 func newTestCache(t *testing.T, upstreamURL string, configure func(*Options)) *testCache {
@@ -347,7 +348,7 @@ func newTestCache(t *testing.T, upstreamURL string, configure func(*Options)) *t
 		server.Close()
 		_ = c.Close()
 	})
-	return &testCache{Cache: c, server: server}
+	return &testCache{Cache: c, server: server, logs: logs}
 }
 
 // testFront serves /git/up/{repo}/info/refs and git-upload-pack from c, as

internal/gitcache/mirror.go

@@ -4,27 +4,47 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os/exec"
+	"slices"
 	"strings"
 	"time"
 )
 
 // MirrorResult is what Mirror did.
 type MirrorResult struct {
-	// Cached reports that the repository, and the commit if one was given,
-	// were cached already.
+	// Cached reports that the repository, and the commit if a version was
+	// given, were cached already.
 	Cached bool
 	// Bytes is how much the cached repository grew.
 	Bytes int64
+	// Commit is the commit the version names, empty without a version.
+	Commit string
+	// Submodules are the submodules Commit records, sorted by path.
+	Submodules []Submodule
+}
+
+// Submodule is a submodule a commit records: its path, the URL .gitmodules
+// gives for it, which may be relative to the repository's, and the commit
+// the gitlink pins.
+type Submodule struct {
+	Path, URL, Commit string
 }
 
 // Mirror brings the repository cached under key up to date with repoURL,
-// cloning it if needed, and waits for that to finish. With commit, an object
-// ID, it also makes sure the commit is cached. One on no branch or tag is
-// fetched by ID when fetchWants allows it (see OpenOptions.FetchWants), and
-// is kept as long as a commit a client fetched by ID.
-func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWants bool) (MirrorResult, error) {
-	if commit != "" && !isObjectID(commit) {
-		return MirrorResult{}, fmt.Errorf("%q is not a full commit ID", commit)
+// cloning it if needed, and waits for that to finish. version, if given, is
+// a full commit ID or a tag. A commit is made sure to be cached: one on no
+// branch or tag is fetched by ID when fetchWants allows it (see
+// OpenOptions.FetchWants), and is kept as long as a commit a client fetched
+// by ID. A tag is cached with the repository's tags.
+func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchWants bool) (MirrorResult, error) {
+	if lower := strings.ToLower(version); isObjectID(lower) {
+		version = lower
+	}
+	tag := version != "" && !isObjectID(version)
+	if tag {
+		if err := c.checkTagName(ctx, version); err != nil {
+			return MirrorResult{}, err
+		}
 	}
 	lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants, NoCheck: true})
 	if err != nil {
@@ -53,17 +73,132 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWa
 	if errors.Is(refreshErr, ErrTooLarge) {
 		return MirrorResult{}, refreshErr
 	}
-	if commit != "" {
+	commit := version
+	switch {
+	case tag:
+		if commit, err = c.resolveTag(ctx, rp.dir, version); err != nil {
+			if refreshErr != nil {
+				// The tag may be one the failed refresh would have brought.
+				return MirrorResult{}, refreshErr
+			}
+			if errors.Is(err, ErrNotFound) && strings.Trim(strings.ToLower(version), "0123456789abcdef") == "" {
+				// Likely an abbreviated commit ID.
+				return MirrorResult{}, fmt.Errorf("%s is not a full commit ID or tag: %w", version, ErrNotFound)
+			}
+			return MirrorResult{}, err
+		}
+		if refreshErr != nil {
+			// Tags can move; this is where it pointed when last fetched.
+			c.logger.Warn("git cache: refresh failed, mirroring the cached tag", "repository", key, "tag", version, "commit", commit, "error", refreshErr)
+		}
+	case version != "":
 		if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil {
 			return MirrorResult{}, err
 		}
 	}
 	// A refresh that failed leaves the commit cached all the same.
-	if refreshErr != nil && commit == "" {
+	if refreshErr != nil && version == "" {
 		return MirrorResult{}, refreshErr
 	}
+	var submodules []Submodule
+	if commit != "" {
+		if submodules, err = c.submodules(ctx, key, rp.dir, commit); err != nil {
+			return MirrorResult{}, err
+		}
+	}
 	grown := max(rp.size.Load()-before, 0)
-	return MirrorResult{Cached: cached && grown == 0, Bytes: grown}, nil
+	return MirrorResult{Cached: cached && grown == 0, Bytes: grown, Commit: commit, Submodules: submodules}, nil
+}
+
+// checkTagName refuses a version that is no valid tag name, such as one
+// with .. or a revision suffix like ^{tree}.
+func (c *Cache) checkTagName(ctx context.Context, name string) error {
+	if err := c.command(ctx, nil, "check-ref-format", "refs/tags/"+name).Run(); err != nil {
+		return fmt.Errorf("%q is neither a full commit ID nor a tag name", name)
+	}
+	return nil
+}
+
+// resolveTag returns the commit tag points at.
+func (c *Cache) resolveTag(ctx context.Context, dir, tag string) (string, error) {
+	out, err := c.command(ctx, nil, "-C", dir, "rev-parse", "--verify", "--quiet", "--end-of-options", "refs/tags/"+tag+"^{commit}").Output()
+	var exit *exec.ExitError
+	if errors.As(err, &exit) && exit.ExitCode() == 1 {
+		return "", fmt.Errorf("tag %s: %w", tag, ErrNotFound)
+	}
+	if err != nil {
+		return "", fmt.Errorf("resolving tag %s: %w", tag, err)
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
+// submodules lists the submodules commit records: its gitlinks that
+// .gitmodules names a URL for.
+func (c *Cache) submodules(ctx context.Context, key, dir, commit string) ([]Submodule, error) {
+	err := c.command(ctx, nil, "-C", dir, "rev-parse", "--verify", "--quiet", "--end-of-options", commit+":.gitmodules").Run()
+	var exit *exec.ExitError
+	if errors.As(err, &exit) && exit.ExitCode() == 1 && ctx.Err() == nil {
+		return nil, nil
+	}
+	if err != nil {
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+		return nil, fmt.Errorf("looking for .gitmodules: %w", err)
+	}
+	// --list rather than --get-regexp, which fails when nothing matches.
+	out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--list")
+	if err != nil {
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+		// The commit is cached; a .gitmodules git can't read only loses
+		// its submodules.
+		c.logger.Warn("git cache: cannot read .gitmodules, skipping submodules", "repository", key, "commit", commit, "error", err)
+		return nil, nil
+	}
+	paths, urls := map[string]string{}, map[string]string{}
+	for entry := range strings.SplitSeq(out, "\x00") {
+		key, value, ok := strings.Cut(entry, "\n")
+		section, ok2 := strings.CutPrefix(key, "submodule.")
+		name, field, dotted := strings.CutLast(section, ".")
+		if !ok || !ok2 || !dotted {
+			continue
+		}
+		switch field {
+		case "path":
+			paths[value] = name
+		case "url":
+			urls[name] = value
+		}
+	}
+	var pathspecs []string
+	for path, name := range paths {
+		if path != "" && urls[name] != "" {
+			pathspecs = append(pathspecs, path)
+		}
+	}
+	if len(pathspecs) == 0 {
+		return nil, nil
+	}
+	args := append([]string{"--literal-pathspecs", "ls-tree", "-z", "--full-tree", commit, "--"}, pathspecs...)
+	tree, err := c.run(ctx, dir, args...)
+	if err != nil {
+		return nil, err
+	}
+	var submodules []Submodule
+	for entry := range strings.SplitSeq(tree, "\x00") {
+		meta, path, ok := strings.Cut(entry, "\t")
+		fields := strings.Fields(meta)
+		if !ok || len(fields) != 3 || fields[0] != "160000" {
+			continue
+		}
+		if url := urls[paths[path]]; url != "" && isObjectID(fields[2]) {
+			submodules = append(submodules, Submodule{Path: path, URL: url, Commit: fields[2]})
+		}
+	}
+	slices.SortFunc(submodules, func(a, b Submodule) int { return strings.Compare(a.Path, b.Path) })
+	return submodules, nil
 }
 
 // ensureCommit makes sure the repository has commit, fetching it by ID when