fix(container-cache): run the ownership init container from the certificates image

deploy/helm/container-cache/README.md

@@ -273,6 +273,7 @@ Key behaviors:
 | `images.server` | `<your-registry>/<your-org>/nvcf-container-cache:v1.1.33` | Cache proxy image |
 | `images.exporter` | `<your-registry>/<your-org>/nginx-prometheus-exporter:1.0` | Metrics exporter |
 | `images.certificates` | `<your-registry>/<your-org>/nvcf-proxy-tls-certs:1.2.2` | TLS cert bundle |
+| `images.init` | `""` (uses `images.certificates`) | Init image that fixes cache volume ownership; needs bash, chown, chmod, find |
 | `images.secrets` | `[nvidia-ngcuser-pull-secret]` | Image pull secrets |
 
 ### Cache Behavior

deploy/helm/container-cache/deploy/templates/statefulset.yaml

@@ -65,7 +65,11 @@ spec:
             topologyKey: kubernetes.io/hostname
       initContainers:
         - name: init-chown-data
-          image: nvcr.io/nvidia/cuda:12.3.2-base-ubuntu22.04
+          # Only bash, chown, chmod and find are needed, so the certificates
+          # image (same registry org and pull secret as every other image in
+          # this pod) is the default. A public base image here failed to pull
+          # once the pod carried an org-scoped pull secret.
+          image: {{ $.Values.images.init | default $.Values.images.certificates | quote }}
           command: [ "/bin/bash" ]
           args:
             - -c
@@ -75,7 +79,9 @@ spec:
             # recursive chown/chmod below skips .snapshot: some network
             # filesystems reserve that directory at the mount root, and
             # chown -R into it hangs/fails, crash-looping this init container.
-            - "chmod 755 /proxy_cache /container_cache && chown 1000:1000 /proxy_cache /container_cache && find /proxy_cache /container_cache -mindepth 1 -maxdepth 1 ! -name .snapshot -exec chown -R 1000:1000 {} + -exec chmod -R u+rwX {} +"
+            # PATH puts the distribution coreutils ahead of any busybox
+            # symlinks the image may carry under /usr/local/bin.
+            - "export PATH=/usr/bin:/bin:$PATH && chmod 755 /proxy_cache /container_cache && chown 1000:1000 /proxy_cache /container_cache && find /proxy_cache /container_cache -mindepth 1 -maxdepth 1 ! -name .snapshot -exec chown -R 1000:1000 {} + -exec chmod -R u+rwX {} +"
           resources: {}
           volumeMounts:
             - name: proxy-cache

deploy/helm/container-cache/deploy/values.yaml

@@ -37,6 +37,10 @@ images:
   exporter: nvcr.io/nv-ngc-devops/nginx-prometheus-exporter:1.0
   # Certificate bundle used for cache certificates.
   certificates: nvcr.io/nv-ngc-devops/nvcf-proxy-tls-certs:v1.2.11
+  # Image for the init container that fixes ownership of the cache volumes.
+  # Needs only bash, chown, chmod and find. Empty means the certificates image,
+  # which is pulled from the same registry org with the same pull secret.
+  init: ""
   # Optional pull secrets for private registries.
   secrets:
     - nvidia-ngcuser-pull-secret

deploy/helm/container-cache/tests/chart-render/verify-init-image.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Every image the chart runs must come from values, so one registry org and one
+# pull secret cover the whole pod. A hard-coded public base image for the
+# ownership init container failed to pull once the pod carried an org-scoped
+# pull secret and left the StatefulSet stuck mid-rollout.
+# Run from deploy/helm/container-cache.
+set -euo pipefail
+
+OUT_FILE="$(mktemp)"
+trap 'rm -f "${OUT_FILE}"' EXIT
+
+assert_has() {
+  if ! grep -F -q -- "$1" "${OUT_FILE}"; then
+    echo "FAILED: expected pattern not found: $1" >&2
+    exit 1
+  fi
+}
+
+assert_not_has() {
+  if grep -F -q -- "$1" "${OUT_FILE}"; then
+    echo "FAILED: unexpected pattern found: $1" >&2
+    exit 1
+  fi
+}
+
+echo "Checking no image is hard-coded to a public registry org..."
+helm template c ./deploy > "${OUT_FILE}"
+assert_not_has 'nvcr.io/nvidia/'
+
+echo "Checking the init container defaults to the certificates image..."
+helm template c ./deploy --set images.certificates=reg.example/org/certs:9 > "${OUT_FILE}"
+# Two containers run from that image: init-chown-data and certs.
+count=$(grep -F -c 'image: "reg.example/org/certs:9"' "${OUT_FILE}" || true)
+count=$((count + $(grep -F -c 'image:  reg.example/org/certs:9' "${OUT_FILE}" || true)))
+if [ "${count}" -lt 2 ]; then
+  echo "FAILED: expected init-chown-data and certs to both use the certificates image, found ${count}" >&2
+  exit 1
+fi
+
+echo "Checking the init image is overridable..."
+helm template c ./deploy --set images.init=reg.example/org/init:1 > "${OUT_FILE}"
+assert_has 'image: "reg.example/org/init:1"'
+
+echo "Checking the init command prefers distribution coreutils..."
+assert_has 'export PATH=/usr/bin:/bin:$PATH && chmod 755 /proxy_cache /container_cache'
+
+echo "PASS: init image render checks"