fix(release): pull the infra images' base images from mirror.gcr.io in CI

.github/scripts/mirror_base_images.py

@@ -0,0 +1,94 @@
+"""Pull the Docker official images that Dockerfiles build from on public.ecr.aws from mirror.gcr.io instead, under the
+names the Dockerfiles use, so their builds don't pull them from public.ecr.aws.
+
+ECR Public rate-limits anonymous pulls by source IP, and hosted runners share IPs, so its pulls fail for hours at a time.
+mirror.gcr.io serves the same images under the same digests, but keeps one only while it's asked for, so a base it
+doesn't give is left to its build to pull from public.ecr.aws.
+
+Docker 28 resolves a digest-pinned FROM only from an image under that exact name, which ``docker tag`` refuses, so those
+are also named in containerd, whose images Docker reads only on the containerd image store. Switching dockerd to it
+restarts dockerd, which stops the containers it runs, so this runs before any start.
+
+``mirror_base_images.py PATH...`` does this for the Dockerfiles at or under each PATH.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+ECR = "public.ecr.aws/docker/library/"
+MIRROR = "mirror.gcr.io/library/"
+_FROM = re.compile(rf"^FROM\s+(?:--\S+\s+)*({re.escape(ECR)}\S+)", re.IGNORECASE | re.MULTILINE)
+_PULL_WAITS_SECONDS = (10, 20)
+_DAEMON_CONFIG = Path("/etc/docker/daemon.json")
+_CONTAINERD_STORE = "io.containerd.snapshotter"
+
+
+def main(argv: list[str]) -> int:
+    bases = _bases([Path(arg) for arg in argv])
+    if any("@" in base for base in bases):
+        _use_containerd_store()
+    for base in bases:
+        _pull(base)
+    return 0
+
+
+def _bases(paths: list[Path]) -> list[str]:
+    """The public.ecr.aws images the Dockerfiles at or under ``paths`` build from."""
+    dockerfiles = [found for path in paths for found in ([path] if path.is_file() else sorted(path.rglob("Dockerfile*")))]
+    return sorted({base for dockerfile in dockerfiles if dockerfile.is_file()
+                   for base in _FROM.findall(dockerfile.read_text())})
+
+
+def _mirrored(base: str) -> str:
+    """What mirror.gcr.io serves for ``base``: the image under its digest when it's pinned to one, else under its tag."""
+    name, _, digest = base.removeprefix(ECR).partition("@")
+    return f"{MIRROR}{name.split(':')[0]}@{digest}" if digest else f"{MIRROR}{name}"
+
+
+def _pull(base: str) -> None:
+    source = _mirrored(base)
+    for wait in (*_PULL_WAITS_SECONDS, None):
+        if subprocess.run(["docker", "pull", "--quiet", source]).returncode == 0:
+            break
+        if wait is None:
+            print(f"::warning::couldn't pull {source}, so {base}'s build will pull it", flush=True)
+            return
+        time.sleep(wait)
+    name, _, digest = base.partition("@")
+    if not digest or ":" in name.rsplit("/", 1)[-1]:
+        _run(["docker", "tag", source, name])
+    if digest:
+        _run(["sudo", "ctr", "--namespace", "moby", "images", "tag", "--force", source, base])
+    print(f"{base} is {source}", flush=True)
+
+
+def _use_containerd_store() -> None:
+    """Switch dockerd to the containerd image store, which restarts it, unless it's on that store already."""
+    if _CONTAINERD_STORE in _driver_status():
+        return
+    config = json.loads(_DAEMON_CONFIG.read_text().strip() or "{}") if _DAEMON_CONFIG.exists() else {}
+    config.setdefault("features", {})["containerd-snapshotter"] = True
+    subprocess.run(["sudo", "tee", str(_DAEMON_CONFIG)], input=json.dumps(config, indent=2), text=True,
+                   stdout=subprocess.DEVNULL, check=True)
+    _run(["sudo", "systemctl", "restart", "docker"])
+    if _CONTAINERD_STORE not in _driver_status():
+        raise SystemExit("dockerd isn't on the containerd image store after restarting")
+
+
+def _driver_status() -> str:
+    return subprocess.run(["docker", "info", "--format", "{{json .DriverStatus}}"], capture_output=True, text=True,
+                          check=True).stdout
+
+
+def _run(argv: list[str]) -> None:
+    subprocess.run(argv, check=True)
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

.github/workflows/local-backends.yml

@@ -131,39 +131,14 @@ jobs:
           python -m pip install --quiet -r .github/ci-requirements.txt
           uv sync --locked --extra dev --python "$(command -v python)"
           echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
-      # Public ECR caps anonymous pulls by data per source IP, and hosted runners share IPs, so its pulls
-      # fail here for hours at a time. mirror.gcr.io serves the same Docker official images under the same
-      # digests: each base image is pulled from there and tagged with the name its Dockerfile uses. Docker
-      # 28 resolves a digest-pinned FROM only from an image under that exact name, which `docker tag`
-      # refuses, so those are also named in containerd, whose images Docker reads only on the containerd
-      # image store. Switching stores restarts dockerd, which would stop a service container, so the
-      # registry starts afterwards.
-      - name: Use the containerd image store
-        run: |
-          [ -s /etc/docker/daemon.json ] || echo '{}' | sudo tee /etc/docker/daemon.json >/dev/null
-          jq '.features["containerd-snapshotter"] = true' /etc/docker/daemon.json > "$RUNNER_TEMP/daemon.json"
-          sudo mv "$RUNNER_TEMP/daemon.json" /etc/docker/daemon.json
-          sudo systemctl restart docker
-          docker info --format '{{json .DriverStatus}}' | grep -q io.containerd.snapshotter
+      # ECR Public's anonymous pulls fail here for hours at a time. This can restart dockerd, so the registry
+      # starts afterwards.
+      - name: Pull the base images from mirror.gcr.io
+        run: python3 .github/scripts/mirror_base_images.py src tst
       - name: Start the local registry
         run: |
           docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2
           timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done'
-      - name: Pull the base images from mirror.gcr.io
-        run: |
-          grep -rhoE '^FROM +public\.ecr\.aws/docker/library/[^ ]+' --include='Dockerfile*' src tst | awk '{print $2}' | sort -u |
-            while read -r image; do
-              name=${image#public.ecr.aws/docker/library/}
-              source=mirror.gcr.io/library/$name
-              case $name in *@*) source=mirror.gcr.io/library/${name%%[:@]*}@${name#*@} ;; esac
-              for attempt in 1 2 3; do
-                docker pull --quiet "$source" && break
-                if [ "$attempt" = 3 ]; then echo "::warning::couldn't pull $source; $image's build will pull it"; continue 2; fi
-                sleep $((attempt * 10))
-              done
-              case ${name%@*} in *:*) docker tag "$source" "${image%@*}" ;; esac
-              case $image in *@*) sudo ctr --namespace moby images tag "$source" "$image" >/dev/null ;; esac
-            done
       - name: Slow integration tier on the local defaults
         run: |
           .venv/bin/python -m pytest -m int_test_slow -rA --timeout=1200 --junitxml=junit-integration-slow.xml \

.github/workflows/publish-pypi.yml

@@ -125,6 +125,9 @@ jobs:
             1) echo "build=true" >> "$GITHUB_OUTPUT" ;;
             *) exit 1 ;;
           esac
+      - name: Pull its base images from mirror.gcr.io, as public.ecr.aws can rate-limit a runner for hours
+        if: steps.wanted.outputs.build == 'true'
+        run: python3 .github/scripts/mirror_base_images.py "$DOCKERFILE"
       - name: Build
         if: steps.wanted.outputs.build == 'true'
         run: |

tst/unit/test_mirror_base_images_script.py

@@ -0,0 +1,141 @@
+"""``.github/scripts/mirror_base_images.py``, which pulls the base images Dockerfiles name on public.ecr.aws from
+mirror.gcr.io before a workflow builds them, falling back to public.ecr.aws for one the mirror doesn't give."""
+
+from __future__ import annotations
+
+import ast
+import json
+import subprocess
+import sys
+
+import pytest
+
+from agent_env.env import bootstrap
+from tst.util.scripts import SCRIPTS, load_script
+
+REPO = SCRIPTS.parents[1]
+SCRIPT = SCRIPTS / "mirror_base_images.py"
+_DIGEST = "sha256:" + "c" * 64
+_PINNED = f"public.ecr.aws/docker/library/python:3.11-slim@{_DIGEST}"
+_TAGGED = "public.ecr.aws/docker/library/postgres:16-alpine"
+
+
+@pytest.fixture
+def script():
+    return load_script(SCRIPT.name)
+
+
+def test_it_imports_only_the_standard_library():
+    """The release's image builds run it in a job that holds a token to push, which installs nothing."""
+    imported = set()
+    for node in ast.walk(ast.parse(SCRIPT.read_text())):
+        if isinstance(node, ast.Import):
+            imported |= {alias.name.split(".")[0] for alias in node.names}
+        elif isinstance(node, ast.ImportFrom) and node.level == 0:
+            imported.add(node.module.split(".")[0])
+
+    assert imported - {"__future__"} <= sys.stdlib_module_names, imported - sys.stdlib_module_names
+
+
+def test_it_finds_each_public_ecr_base_once(script, tmp_path):
+    (tmp_path / "a").mkdir()
+    (tmp_path / "a" / "Dockerfile").write_text(f"FROM {_PINNED}\nRUN true\nFROM ghcr.io/astral-sh/uv:latest AS uv\n")
+    (tmp_path / "a" / "Dockerfile.db-mcp").write_text(f"from --platform=$BUILDPLATFORM {_TAGGED} AS build\n")
+    (tmp_path / "b").mkdir()
+    (tmp_path / "b" / "Dockerfile").write_text(f"# FROM {_TAGGED}:old\nFROM {_PINNED}\n")
+    (tmp_path / "b" / "README.md").write_text("FROM public.ecr.aws/docker/library/node:22\n")
+
+    assert script._bases([tmp_path]) == [_TAGGED, _PINNED]
+    assert script._bases([tmp_path / "a" / "Dockerfile.db-mcp"]) == [_TAGGED]
+
+
+def test_it_finds_the_bases_of_the_images_agent_env_ships(script):
+    bases = script._bases([image.dockerfile for image in bootstrap.PUBLISHED_IMAGES])
+
+    assert bases and all(base.startswith(script.ECR) for base in bases)
+    assert any("@sha256:" in base for base in bases)
+
+
+@pytest.mark.parametrize("base, mirrored", [
+    (_PINNED, f"mirror.gcr.io/library/python@{_DIGEST}"),
+    (_TAGGED, "mirror.gcr.io/library/postgres:16-alpine"),
+])
+def test_a_base_is_pulled_by_its_digest_when_it_has_one(script, base, mirrored):
+    assert script._mirrored(base) == mirrored
+
+
+def _docker(monkeypatch, script, *, store="overlay2", pulls_fail=0):
+    """Fakes the commands the script runs, recording them; the first ``pulls_fail`` pulls fail."""
+    ran, failures, driver = [], iter(range(pulls_fail)), [store]
+
+    def run(argv, **kwargs):
+        ran.append(argv)
+        if argv[:2] == ["docker", "info"]:
+            return subprocess.CompletedProcess(argv, 0, json.dumps([["driver-type", driver[0]]]), "")
+        if argv[:2] == ["sudo", "systemctl"]:
+            driver[0] = "io.containerd.snapshotter.v1"
+        if argv[:2] == ["sudo", "tee"]:
+            ran.append(json.loads(kwargs["input"]))
+        failed = argv[:2] == ["docker", "pull"] and next(failures, None) is not None
+        return subprocess.CompletedProcess(argv, 1 if failed else 0, "", "")
+
+    monkeypatch.setattr(script.subprocess, "run", run)
+    monkeypatch.setattr(script.time, "sleep", lambda seconds: ran.append(["sleep", seconds]))
+    return ran
+
+
+def test_a_pinned_base_is_named_in_containerd_on_its_image_store(script, monkeypatch, tmp_path):
+    config = tmp_path / "daemon.json"
+    config.write_text('{"exec-opts": ["native.cgroupdriver=cgroupfs"]}\n')
+    monkeypatch.setattr(script, "_DAEMON_CONFIG", config)
+    ran = _docker(monkeypatch, script)
+    dockerfile = tmp_path / "Dockerfile"
+    dockerfile.write_text(f"FROM {_PINNED}\n")
+
+    assert script.main([str(dockerfile)]) == 0
+
+    mirrored = f"mirror.gcr.io/library/python@{_DIGEST}"
+    assert ran == [
+        ["docker", "info", "--format", "{{json .DriverStatus}}"],
+        ["sudo", "tee", str(config)],
+        {"exec-opts": ["native.cgroupdriver=cgroupfs"], "features": {"containerd-snapshotter": True}},
+        ["sudo", "systemctl", "restart", "docker"],
+        ["docker", "info", "--format", "{{json .DriverStatus}}"],
+        ["docker", "pull", "--quiet", mirrored],
+        ["docker", "tag", mirrored, "public.ecr.aws/docker/library/python:3.11-slim"],
+        ["sudo", "ctr", "--namespace", "moby", "images", "tag", "--force", mirrored, _PINNED],
+    ]
+
+
+def test_dockerd_is_left_alone_when_no_base_is_pinned(script, monkeypatch, tmp_path):
+    ran = _docker(monkeypatch, script)
+    dockerfile = tmp_path / "Dockerfile"
+    dockerfile.write_text(f"FROM {_TAGGED}\n")
+
+    assert script.main([str(dockerfile)]) == 0
+
+    assert ran == [["docker", "pull", "--quiet", "mirror.gcr.io/library/postgres:16-alpine"],
+                   ["docker", "tag", "mirror.gcr.io/library/postgres:16-alpine", _TAGGED]]
+
+
+def test_a_base_the_mirror_doesnt_give_is_left_to_its_build(script, monkeypatch, tmp_path, capsys):
+    ran = _docker(monkeypatch, script, pulls_fail=3)
+    dockerfile = tmp_path / "Dockerfile"
+    dockerfile.write_text(f"FROM {_TAGGED}\n")
+
+    assert script.main([str(dockerfile)]) == 0
+
+    pull = ["docker", "pull", "--quiet", "mirror.gcr.io/library/postgres:16-alpine"]
+    assert ran == [pull, ["sleep", 10], pull, ["sleep", 20], pull]
+    assert f"::warning::couldn't pull mirror.gcr.io/library/postgres:16-alpine, so {_TAGGED}'s build" in capsys.readouterr().out
+
+
+def test_a_pull_that_fails_once_is_tried_again(script, monkeypatch, tmp_path):
+    ran = _docker(monkeypatch, script, pulls_fail=1)
+    dockerfile = tmp_path / "Dockerfile"
+    dockerfile.write_text(f"FROM {_TAGGED}\n")
+
+    assert script.main([str(dockerfile)]) == 0
+
+    assert ran[-1] == ["docker", "tag", "mirror.gcr.io/library/postgres:16-alpine", _TAGGED]
+    assert ["sleep", 10] in ran and ["sleep", 20] not in ran

tst/unit/test_release_infra_images.py

@@ -110,6 +110,17 @@ def test_a_run_by_hand_goes_ahead_only_on_main_or_a_release_tag(job):
     assert "github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')" in JOBS[job]["if"]
 
 
+def test_a_build_pulls_its_base_images_from_the_mirror_first():
+    """public.ecr.aws can rate-limit a hosted runner's anonymous pulls for hours; the build pulls from it only what the
+    mirror doesn't give."""
+    (step,) = [step for step in JOBS["infra-build"]["steps"] if "mirror_base_images.py" in step.get("run", "")]
+    build = "\n".join(_runs("infra-build"))
+
+    assert step["run"] == 'python3 .github/scripts/mirror_base_images.py "$DOCKERFILE"'
+    assert step["if"] == "steps.wanted.outputs.build == 'true'"
+    assert build.index("mirror_base_images.py") < build.index("docker build")
+
+
 def test_what_a_registry_can_rate_limit_is_tried_again():
     """A 429 on a base image's pull would fail the build, and with it the release's PyPI jobs."""
     build = "\n".join(_runs("infra-build"))