Avoid multipart uploads for release artifacts

pythonbuild/mirror.py

@@ -28,6 +28,14 @@
 CACHE_CONTROL = "public, max-age=31536000, immutable"
 UPLOAD_CONCURRENCY = 4
 MAX_ATTEMPTS = 5
+# R2 recommends multipart uploads above approximately 100 MB, but single uploads
+# support objects up to about 5 GiB. Current release artifacts fit under 200 MiB.
+# Single uploads avoid multipart completion for current artifacts, but a failed
+# upload must retransmit the entire file. If R2 stores an object and its response
+# is lost, retrying the same immutable key can still fail. Files at or above this
+# threshold use multipart uploads and require additional retry handling.
+# https://developers.cloudflare.com/r2/objects/upload-objects/#choose-an-upload-method
+SINGLE_UPLOAD_THRESHOLD = 200 * 1024 * 1024
 BUILD_DATETIME_RE = re.compile(r"-(\d{8}T\d{4})\.tar\.(?:gz|zst)$")
 DESTINATION_TAG_RE_TEMPLATE = r"^(cpython-[^+-]+)\+{tag}-(.+)$"
 
@@ -167,7 +175,9 @@ def make_s3_client() -> tuple[Boto3S3Client, TransferConfig]:
 
     session = boto3.session.Session()
     client = session.client("s3", **client_kwargs)
-    transfer_config = TransferConfig()
+    transfer_config = TransferConfig(
+        multipart_threshold=SINGLE_UPLOAD_THRESHOLD,
+    )
 
     return client, transfer_config
 

tests/test_mirror.py

@@ -18,10 +18,20 @@
     build_upload_entries,
     destination_to_source_name,
     main,
+    make_s3_client,
 )
 
 
 class MirrorTests(unittest.TestCase):
+    def test_make_s3_client_uses_single_uploads_below_threshold(self) -> None:
+        with (
+            mock.patch.dict("pythonbuild.mirror.os.environ", {}, clear=True),
+            mock.patch("pythonbuild.mirror.boto3.session.Session"),
+        ):
+            _client, transfer_config = make_s3_client()
+
+        self.assertEqual(transfer_config.multipart_threshold, 200 * 1024 * 1024)
+
     def test_destination_to_source_name_full_archive(self) -> None:
         self.assertEqual(
             destination_to_source_name(

tests/test_mirror_integration.py

@@ -33,7 +33,8 @@ def test_resumes_partial_mirror_upload(self) -> None:
             shasums_path = dist / "SHA256SUMS"
 
             full_path.write_bytes(b"full")
-            install_only_path.write_bytes(b"install")
+            install_contents = b"i" * (8 * 1024 * 1024 + 1)
+            install_only_path.write_bytes(install_contents)
             shasums_path.write_text(
                 "abc  cpython-3.12.10+20260324-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst\n"
                 "def  cpython-3.12.10+20260324-x86_64-unknown-linux-gnu-install_only.tar.gz\n"
@@ -102,7 +103,8 @@ def test_resumes_partial_mirror_upload(self) -> None:
                     Key=prefix
                     + "cpython-3.12.10+20260324-x86_64-unknown-linux-gnu-install_only.tar.gz",
                 )
-                self.assertEqual(install_only_object["Body"].read(), b"install")
+                self.assertEqual(install_only_object["Body"].read(), install_contents)
+                self.assertNotIn("-", install_only_object["ETag"])
                 self.assertEqual(install_only_object["CacheControl"], CACHE_CONTROL)
 
                 shasums_object = s3.get_object(Bucket=bucket, Key=prefix + "SHA256SUMS")