From 74e393697a2dce204dde43cb0c7517eaa8b15ff0 Mon Sep 17 00:00:00 2001 From: Mikko Lauri Date: Wed, 30 Sep 2026 07:52:10 -0500 Subject: [PATCH 1/4] Install ROCm in the CI image from pinned nightly debs. Replace the TheRock source build with runtime, JIT, and devel deb layers so CI no longer compiles ROCm. PyTorch, Triton, and the TheRock checkout pin stay on main; TheRock is cloned only for the PyTorch build scripts. Self-retiring shims cover nightly packaging gaps for amdsmi, ld.lld, and origami. Co-authored-by: Cursor --- docker/Dockerfile.ci | 233 +++++++++--------------------- docker/install_rocm_debs.sh | 94 ++++++++++++ docker/rocm_devel_artifacts.txt | 102 ------------- docker/rocm_runtime_artifacts.txt | 56 ------- docker/setup_amdsmi.sh | 94 ++++++++++++ docker/setup_origami_cmake.sh | 112 ++++++++++++++ docker/setup_rocm_llvm_layout.sh | 69 +++++++++ 7 files changed, 435 insertions(+), 325 deletions(-) create mode 100755 docker/install_rocm_debs.sh delete mode 100644 docker/rocm_devel_artifacts.txt delete mode 100644 docker/rocm_runtime_artifacts.txt create mode 100755 docker/setup_amdsmi.sh create mode 100755 docker/setup_origami_cmake.sh create mode 100755 docker/setup_rocm_llvm_layout.sh diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index d76258c1..74eb32ea 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -13,24 +13,13 @@ ARG BASE_IMAGE="ubuntu:24.04" ARG PYTHON_VERSION="default" ARG THEROCK_COMMIT="ab56f1f7c5b54746bf9f7948fff3f306b2bef8b2" -ARG CCACHE_VERSION="4.11.2" -# AMDGPU_FAMILIES is a ";" separated list of values, choices can be found -# at https://github.com/ROCm/TheRock/blob/main/cmake/therock_amdgpu_targets.cmake -ARG AMDGPU_FAMILIES="gfx94X-dcgpu;gfx950-dcgpu" -# AMDGPU_BUNDLE_NAME disambiguates the cmake-level dist bundle name when more -# than one AMDGPU family is selected. TheRock's therock_validate_amdgpu_targets() -# FATAL_ERRORs if the bundle name cannot be inferred from a single family. -# With KPACK_SPLIT_ARTIFACTS=ON it only appears in the intermediate -# build/artifacts-unsplit/ tree and is not observable in the final image. -ARG AMDGPU_BUNDLE_NAME="gfx94X_gfx950_bundle" +ARG ROCM_RELEASE_ID="20260817-31980779979" +ARG ROCM_DEB_SERIES="10.1" +# ROCM_GFX_TARGETS selects architecture-specific nightly deb package shards. +ARG ROCM_GFX_TARGETS="gfx942;gfx950" # PYTORCH_ROCM_ARCH is a ";" separated list of values, choices for LLVM targets can # be found at https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html#supported-gpus ARG PYTORCH_ROCM_ARCH="gfx942;gfx950" -# Set ROCM_LIBRARIES_COMMIT_OVERRIDE to force to use a specific commit of rocm-libraries monorepo instead of the one pinned in TheRock. -# This is useful for example for testing a feature not yet in the main branch of rocm-libraries or a commit not available via TheRock. -ARG ROCM_LIBRARIES_COMMIT_OVERRIDE="" -# Set ROCM_SYSTEMS_COMMIT_OVERRIDE to force to use a specific commit of rocm-systems monorepo instead of the one pinned in TheRock. -ARG ROCM_SYSTEMS_COMMIT_OVERRIDE="" ARG TRITON_COMMIT="669b31acc1dd1b3fd93286afd5db67f65d9f7557" ARG PYTORCH_COMMIT="3369fa23f831832166651a0b8e151dc2877f3c77" ARG PYTORCH_VISION_COMMIT="53881ee74892a7c22f2b16a84b15a00f64a8bf68" @@ -110,39 +99,16 @@ FROM base_os AS base ARG PYTHON_VERSION ARG DEBIAN_FRONTEND=noninteractive -# install depedencies +# Shared fetch/compiler tools. Native extensions in core (AITER, torchao) +# and the torch/ffmpeg builders still need a host toolchain. RUN apt-get update -qq && apt-get install -y --no-install-recommends \ - automake \ - bison \ build-essential \ ca-certificates \ cmake \ curl \ - flex \ - freeglut3-dev \ - gfortran \ git \ - libgfortran5 \ - libgl1 \ - libgl1-mesa-dev \ - libglib2.0-dev \ - libglu1-mesa-dev \ - libopenblas-dev \ - # Workaround: rocprofiler-sdk SQL output sources include , but - # rocm-systems#6664 no longer propagates bundled SQLite headers to - # rocprofiler-sdk-output-library. Remove once fixed upstream: - # https://github.com/ROCm/TheRock/issues/5868 - libsqlite3-dev \ - # Workaround: profiler-hub generates its bundled SQLite3 amalgamation via - # `make sqlite3.c`, which requires tclsh even with --disable-tcl. Remove - # once fixed upstream: - # https://github.com/ROCm/rocm-systems/issues/10059 - tcl \ - libtool \ ninja-build \ - texinfo \ - wget \ - xxd && \ + wget && \ update-ca-certificates && \ rm -rf /var/lib/apt/lists/* @@ -207,120 +173,71 @@ RUN python3 -m venv venv ENV PATH="/venv/bin:$PATH" RUN pip install --no-cache-dir --upgrade pip packaging setuptools wheel -FROM base AS setup_therock -ARG THEROCK_COMMIT -ARG ROCM_LIBRARIES_COMMIT_OVERRIDE -ARG ROCM_SYSTEMS_COMMIT_OVERRIDE - -WORKDIR /app -RUN git clone https://github.com/ROCm/TheRock.git && \ - cd TheRock && \ - git reset --hard ${THEROCK_COMMIT} && \ - if [ -n "$ROCM_LIBRARIES_COMMIT_OVERRIDE" ]; then \ - echo "Force updating rocm-libraries commit to ${ROCM_LIBRARIES_COMMIT_OVERRIDE}" && \ - git update-index --cacheinfo 160000,${ROCM_LIBRARIES_COMMIT_OVERRIDE},rocm-libraries ; \ - fi && \ - if [ -n "$ROCM_SYSTEMS_COMMIT_OVERRIDE" ]; then \ - echo "Force updating rocm-systems commit to ${ROCM_SYSTEMS_COMMIT_OVERRIDE}" && \ - git update-index --cacheinfo 160000,${ROCM_SYSTEMS_COMMIT_OVERRIDE},rocm-systems ; \ - fi - -FROM base AS build_rocm -ARG CCACHE_VERSION -ARG AMDGPU_FAMILIES -ARG AMDGPU_BUNDLE_NAME -COPY --from=setup_therock /app/TheRock /app/TheRock -WORKDIR /app/TheRock -# Pre-pin meson<1.12.0 before requirements.txt (floor: meson>=1.7.0), mirroring -# TheRock's manylinux Dockerfile. meson 1.12.0 duplicates env-derived link args -# (e.g. LDFLAGS=-Wl,--version-script=...) in its compiler sanity check, which -# `ld` rejects as a duplicate version tag and breaks the libdrm/libpciaccess -# sysdeps builds. See https://github.com/ROCm/TheRock/issues/7286. -RUN pip install --no-cache-dir "meson==1.7.0" -# if dvc is available, fetch_source.py uses it to fetch large files - install it along with other requirements. -# Mirroring TheRock's manylinux Dockerfile pin of aiobotocore so dvc[s3]'s -# botocore window stays aligned with the boto3 range pinned in -# requirements.txt (>=1.42.79,<1.42.85). aiobotocore==3.4.0 requires -# botocore>=1.42.79,<1.42.85; resolving everything in one pip call lets -# pip pick a botocore (1.42.84) that satisfies both pins. -RUN pip install --no-cache-dir -r requirements.txt "pathspec<0.13.0" "dvc[s3]==3.67.1" "aiobotocore==3.4.0" -RUN bash dockerfiles/install_ccache.sh ${CCACHE_VERSION} -RUN bash dockerfiles/install_pinned_patchelf.sh -RUN --mount=type=cache,target=/app/TheRock/.ccache,id=therock-ccache,sharing=locked \ - --mount=type=cache,target=/app/TheRock/git-mirror,id=therock-git-mirrors,sharing=locked \ - python build_tools/setup_git_mirrors.py --mirror-dir /app/TheRock/git-mirror --update --jobs 8 && \ - python build_tools/fetch_sources.py --jobs 8 --depth 1 --reference-dir /app/TheRock/git-mirror && \ - eval "$(python build_tools/setup_ccache.py --init --local)" && \ - ccache -o max_size=50G && \ - ccache -o sloppiness=include_file_ctime,include_file_mtime && \ - ccache -o compiler_check=content && \ - cmake -S /app/TheRock -B /app/TheRock/build -GNinja . \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_HIP_COMPILER_LAUNCHER=ccache \ - -DTHEROCK_AMDGPU_FAMILIES="${AMDGPU_FAMILIES}" \ - -DTHEROCK_TEST_AMDGPU_FAMILIES="${AMDGPU_FAMILIES}" \ - -DTHEROCK_BUNDLE_SYSDEPS=ON -DTHEROCK_AMDGPU_DIST_BUNDLE_NAME="${AMDGPU_BUNDLE_NAME}" \ - -DTHEROCK_ENABLE_ALL=OFF \ - -DTHEROCK_ENABLE_CORE=ON \ - -DTHEROCK_ENABLE_COMM_LIBS=ON \ - -DTHEROCK_ENABLE_DEBUG_TOOLS=ON \ - -DTHEROCK_ENABLE_MATH_LIBS=ON \ - -DTHEROCK_ENABLE_ML_LIBS=ON \ - -DTHEROCK_ENABLE_PROFILER=ON \ - -DTHEROCK_ENABLE_DC_TOOLS=ON \ - -DTHEROCK_ENABLE_MEDIA_LIBS=ON && \ - cmake --build /app/TheRock/build --target therock-artifacts && \ - echo "CCache stats:" && ccache -s -v && \ - mv /app/TheRock/build/artifacts /app/TheRock/artifacts -COPY .ci/collect_image_licenses.py /usr/local/libexec/collect_image_licenses.py -RUN python3 /usr/local/libexec/collect_image_licenses.py \ - --output /license-evidence \ - --scan-root therock=/app/TheRock \ - --git-root /app/TheRock \ - --fail-on-missing-git-license \ - --required-git-depth 0 - FROM base AS rocm_runtime -COPY docker/rocm_runtime_artifacts.txt /tmp/ -RUN --mount=type=bind,from=build_rocm,source=/app/TheRock,target=/app/TheRock/ \ - set -eu; \ - A="/app/TheRock/artifacts"; O="/opt/rocm"; \ - PREFIXES=$(sed '/^#/d; /^$/d' /tmp/rocm_runtime_artifacts.txt | tr '\n' ' '); \ - python3 /app/TheRock/build_tools/fileset_tool.py artifact-flatten-split \ - -o "${O}" --artifacts-dir "${A}" ${PREFIXES} -# Install amdsmi Python module into system site-packages. TheRock stages -# sources under share/amd_smi/amdsmi/ without setup.py (packaging rework in -# rocm-systems 5bc651a8+); pip install from share/amd_smi no longer works. -RUN AMDSMI_SITELIB="$(python3 -c "import site; print([p for p in site.getsitepackages() if not p.startswith('/usr/local')][0])")" && \ - cp -r /opt/rocm/share/amd_smi/amdsmi "${AMDSMI_SITELIB}/" -RUN echo "/opt/rocm/lib" > /etc/ld.so.conf.d/rocm.conf && \ - echo "/opt/rocm/lib/rocm_sysdeps/lib/" > /etc/ld.so.conf.d/rocm_sysdeps.conf && \ - ldconfig -ENV PATH="/opt/rocm/bin:$PATH" \ +ARG ROCM_RELEASE_ID +ARG ROCM_DEB_SERIES +ARG ROCM_GFX_TARGETS +COPY docker/install_rocm_debs.sh /usr/local/bin/install_rocm_debs +RUN chmod +x /usr/local/bin/install_rocm_debs && install_rocm_debs runtime +# The nightly deb ships the amd-smi Python bindings somewhere pip cannot reach; +# link them into site-packages. Prints a RETIRE banner and does nothing once a +# nightly makes `import amdsmi` work on its own. +COPY docker/setup_amdsmi.sh /usr/local/bin/setup_amdsmi +RUN chmod +x /usr/local/bin/setup_amdsmi && setup_amdsmi +ENV PATH="/opt/rocm/lib/llvm/bin:/opt/rocm/bin:$PATH" \ ROCM_HOME=/opt/rocm \ ROCM_PATH=/opt/rocm \ + HIP_CLANG_PATH=/opt/rocm/lib/llvm/bin \ HIP_DEVICE_LIB_PATH=/opt/rocm/lib/llvm/amdgcn/bitcode +# Nightly debs leave $ROCM_PATH/llvm/bin as a stub; FlyDSL/MLIR still +# invoke ld.lld there. Prints a RETIRE banner and does nothing once a +# nightly ships that path itself. Also retire if FlyDSL stops looking +# there. PATH is the ENV above, not this shim. +COPY docker/setup_rocm_llvm_layout.sh /usr/local/bin/setup_rocm_llvm_layout +RUN chmod +x /usr/local/bin/setup_rocm_llvm_layout && setup_rocm_llvm_layout -FROM rocm_runtime AS rocm_devel -COPY docker/rocm_devel_artifacts.txt /tmp/ -RUN --mount=type=bind,from=build_rocm,source=/app/TheRock,target=/app/TheRock/ \ - set -eu; \ - A="/app/TheRock/artifacts"; O="/opt/rocm"; \ - PREFIXES=$(sed '/^#/d; /^$/d' /tmp/rocm_devel_artifacts.txt | tr '\n' ' '); \ - python3 /app/TheRock/build_tools/fileset_tool.py artifact-flatten-split \ - -o "${O}" --artifacts-dir "${A}" ${PREFIXES} -RUN ldconfig +FROM rocm_runtime AS rocm_runtime_jit +ARG ROCM_RELEASE_ID +ARG ROCM_DEB_SERIES +ARG ROCM_GFX_TARGETS +RUN install_rocm_debs jit +# hipBLASLt's CMake config find_dependency(origami) next to hipblaslt; +# nightly debs ship liborigami.so without lib/cmake/origami. Prints a +# RETIRE banner and does nothing once a nightly ships that package. +COPY docker/setup_origami_cmake.sh /usr/local/bin/setup_origami_cmake +RUN chmod +x /usr/local/bin/setup_origami_cmake && setup_origami_cmake +ENV TRITON_HIP_LLD_PATH=/opt/rocm/lib/llvm/bin/ld.lld + +FROM rocm_runtime_jit AS rocm_devel +ARG ROCM_RELEASE_ID +ARG ROCM_DEB_SERIES +ARG ROCM_GFX_TARGETS +RUN install_rocm_debs devel ENV PATH="/opt/rocm/libexec/rocprofiler-compute:$PATH" FROM rocm_devel AS build_torch_stack +ARG THEROCK_COMMIT ARG TRITON_COMMIT ARG PYTORCH_COMMIT ARG PYTORCH_VISION_COMMIT ARG PYTORCH_AUDIO_COMMIT ARG PYTORCH_CODEC_COMMIT ARG PYTORCH_ROCM_ARCH -COPY --from=setup_therock /app/TheRock /app/TheRock +ARG DEBIAN_FRONTEND=noninteractive + +# CPU BLAS headers for the PyTorch wheel. Runtime libs are installed in core. +RUN apt-get update -qq && apt-get install -y --no-install-recommends \ + libopenblas-dev \ + pkg-config && \ + rm -rf /var/lib/apt/lists/* + +# TheRock is only needed for external-builds/pytorch checkout scripts. +WORKDIR /app +RUN git init TheRock && \ + cd TheRock && \ + git remote add origin https://github.com/ROCm/TheRock.git && \ + git fetch --depth 1 origin ${THEROCK_COMMIT} && \ + git checkout --detach FETCH_HEAD # needed for HIPIFY commits created by the build process RUN git config --global user.name "TheRock Builder" && \ @@ -453,25 +370,23 @@ RUN python3 /usr/local/libexec/collect_image_licenses.py \ --output /ffmpeg/license-evidence \ --scan-root ffmpeg=${FFMPEG_SRC_DIR} -FROM rocm_runtime AS core_runtime +# Lean cutover: change this parent to rocm_runtime_jit. +FROM rocm_devel AS core ARG AITER_COMMIT RUN --mount=type=bind,from=build_torch_stack,source=/install,target=/install \ pip install --no-cache-dir /install/*.whl -# AITER JIT: install dev artifacts required for runtime kernel compilation -RUN --mount=type=bind,from=build_rocm,source=/app/TheRock,target=/app/TheRock/ \ - set -eu; \ - A="/app/TheRock/artifacts"; O="/opt/rocm"; \ - python3 /app/TheRock/build_tools/fileset_tool.py artifact-flatten-split \ - -o "${O}" --artifacts-dir "${A}" \ - base_dev core-hip_dev core-runtime_dev blas_dev prim_dev rand_dev -# torchcodec: install FFmpeg and runtime dependencies +# Runtime libs for torchcodec/FFmpeg, OpenBLAS-linked torch, and OpenCV/GLib. RUN apt-get update -qq && apt-get -y install --no-install-recommends \ libaom3 \ libass9 \ libdav1d7 \ libfreetype6 \ + libgfortran5 \ + libgl1 \ + libglib2.0-0t64 \ libgnutls30 \ libmp3lame0 \ + libopenblas0 \ libopenh264-7 \ libvorbis0a \ libvorbisenc2 \ @@ -485,8 +400,7 @@ COPY --from=ffmpeg_builder /ffmpeg/build/share /usr/local/share RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/ffmpeg.conf && \ ldconfig COPY docker/libkineto.conf /config/libkineto.conf -ENV TRITON_HIP_LLD_PATH=/opt/rocm/lib/llvm/bin/ld.lld \ - KINETO_CONFIG="/config/libkineto.conf" +ENV KINETO_CONFIG="/config/libkineto.conf" # install aiter WORKDIR /app/external # Keep the Triton wheel we built from TRITON_COMMIT against our PyTorch; otherwise @@ -502,22 +416,8 @@ RUN --mount=type=bind,source=patches,target=/patches \ git submodule update --init --recursive && \ pip install --no-cache-dir -r requirements.txt && \ pip install -e . - -FROM core_runtime AS core -# Layer full devel artifacts on top of core_runtime (overlapping JIT-dev -# archives are re-extracted idempotently). Uses the same manifest as rocm_devel. -COPY docker/rocm_devel_artifacts.txt /tmp/ -RUN --mount=type=bind,from=build_rocm,source=/app/TheRock,target=/app/TheRock/ \ - set -eu; \ - A="/app/TheRock/artifacts"; O="/opt/rocm"; \ - PREFIXES=$(sed '/^#/d; /^$/d' /tmp/rocm_devel_artifacts.txt | tr '\n' ' '); \ - python3 /app/TheRock/build_tools/fileset_tool.py artifact-flatten-split \ - -o "${O}" --artifacts-dir "${A}" ${PREFIXES} -RUN ldconfig -ENV PATH="/opt/rocm/libexec/rocprofiler-compute:$PATH" # Held in core rather than final so that no stage downstream of core has an edge # back into a builder stage. -COPY --from=build_rocm /license-evidence /opt/license-evidence/rocm-build-sources COPY --from=build_torch_stack /install/license-evidence /opt/license-evidence/torch-build-sources COPY --from=ffmpeg_builder /ffmpeg/license-evidence /opt/license-evidence/ffmpeg-build-sources @@ -633,7 +533,6 @@ RUN python3 /usr/local/libexec/collect_image_licenses.py \ --scan-root python=/venv \ --scan-root rocm=/opt/rocm \ --scan-root local=/usr/local \ - --evidence-root rocm-build-sources=/opt/license-evidence/rocm-build-sources \ --evidence-root torch-build-sources=/opt/license-evidence/torch-build-sources \ --evidence-root ffmpeg-build-sources=/opt/license-evidence/ffmpeg-build-sources \ --git-root /app \ diff --git a/docker/install_rocm_debs.sh b/docker/install_rocm_debs.sh new file mode 100755 index 00000000..446a0f73 --- /dev/null +++ b/docker/install_rocm_debs.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +set -euo pipefail + +readonly layer="${1:?usage: install_rocm_debs runtime|jit|devel}" +readonly release_id="${ROCM_RELEASE_ID:?ROCM_RELEASE_ID must be set}" +readonly deb_series="${ROCM_DEB_SERIES:?ROCM_DEB_SERIES must be set}" +readonly gfx_targets="${ROCM_GFX_TARGETS:?ROCM_GFX_TARGETS must be set}" +readonly rocm_root="${ROCM_HOME:-/opt/rocm}" +readonly repo_url="https://rocm.nightlies.amd.com/packages-multi-arch/deb/${release_id}" + +case "${layer}" in + runtime|jit|devel) ;; + *) + echo "unknown layer '${layer}'; expected runtime, jit, or devel" >&2 + exit 2 + ;; +esac + +echo "deb [trusted=yes] ${repo_url} stable main" \ + > /etc/apt/sources.list.d/rocm-nightly.list + +IFS=';' read -r -a targets <<< "${gfx_targets}" +if [[ "${#targets[@]}" -eq 0 ]]; then + echo "ROCM_GFX_TARGETS must contain at least one target" >&2 + exit 2 +fi + +packages=() +case "${layer}" in + runtime) + for target in "${targets[@]}"; do + [[ -n "${target}" ]] || continue + packages+=("amdrocm-core${deb_series}-${target}") + done + ;; + jit) + for target in "${targets[@]}"; do + [[ -n "${target}" ]] || continue + packages+=("amdrocm-core-dev${deb_series}-${target}") + done + ;; + devel) + packages=( + "amdrocm-developer-tools${deb_series}" + "amdrocm-rdc${deb_series}" + "amdrocm-opencl${deb_series}" + ) + for target in "${targets[@]}"; do + [[ -n "${target}" ]] || continue + packages+=( + "amdrocm-blas-test${deb_series}-${target}" + "amdrocm-rccl-test${deb_series}-${target}" + ) + done + ;; +esac + +if [[ "${#packages[@]}" -eq 0 ]]; then + echo "ROCM_GFX_TARGETS did not contain a usable target" >&2 + exit 2 +fi + +apt-get update -qq +apt-get install -y --no-install-recommends "${packages[@]}" +rm -rf /var/lib/apt/lists/* + +echo "${rocm_root}/lib" > /etc/ld.so.conf.d/rocm.conf +if [[ -d "${rocm_root}/lib/rocm_sysdeps/lib" ]]; then + echo "${rocm_root}/lib/rocm_sysdeps/lib" \ + > /etc/ld.so.conf.d/rocm_sysdeps.conf +fi +ldconfig + +case "${layer}" in + runtime) + test -d "${rocm_root}/lib/llvm/amdgcn/bitcode" + test -e "${rocm_root}/lib/llvm/bin/clang++" + test -e "${rocm_root}/lib/libamdhip64.so" + test -d "${rocm_root}/lib/rocm_sysdeps/lib" + test -e "${rocm_root}/lib/librocprofiler-sdk.so" + test -e "${rocm_root}/lib/libroctracer64.so" + ;; + jit) + test -d "${rocm_root}/include/roctracer" + test -d "${rocm_root}/lib/rocm_sysdeps/include" + test -d "${rocm_root}/lib/rocm_sysdeps/lib/pkgconfig" + ;; + devel) + test -d "${rocm_root}/libexec/rocprofiler-compute" + ;; +esac diff --git a/docker/rocm_devel_artifacts.txt b/docker/rocm_devel_artifacts.txt deleted file mode 100644 index f0b4b30d..00000000 --- a/docker/rocm_devel_artifacts.txt +++ /dev/null @@ -1,102 +0,0 @@ -# ROCm devel/debug/test artifacts layered on top of the runtime set. -# Shared by the rocm_devel stage (for build_torch_stack) and the core stage. -# Lines starting with '#' are comments; blank lines are ignored. -# Each entry is an artifact prefix (_) consumed by -# fileset_tool.py artifact-flatten-split, which globs both the -# _generic shard and any per-arch shards produced by KPACK splitting. - -amd-dbgapi_dbg -amd-dbgapi_dev -amd-dbgapi_lib -amd-llvm_dbg -amd-llvm_dev -aqlprofile_dbg -aqlprofile_dev -base_dbg -base_dev -blas_dbg -blas_dev -# blas_test artifact is included because of hipblaslt-bench -blas_test -composable-kernel_dbg -composable-kernel_dev -core-amdsmi_dbg -core-amdsmi_dev -core-hip_dbg -core-hip_dev -core-kpack_dbg -core-kpack_dev -core-ocl_dbg -core-ocl_dev -core-ocl_lib -core-ocl_run -core-runtime_dbg -core-runtime_dev -fft_dbg -fft_dev -fftw3_dbg -fftw3_dev -hipblasltprovider_dbg -hipdnn_dbg -hipdnn_dev -hipify_dbg -hipify_run -host-blas_dbg -host-blas_dev -host-suite-sparse_dbg -host-suite-sparse_dev -libhipcxx_dbg -libhipcxx_dev -miopen_dbg -miopen_dev -miopenprovider_dbg -prim_dbg -prim_dev -rand_dbg -rand_dev -rccl_dbg -rccl_dev -# rccl_test artifact is included because of all_gather_perf and similar tools -rccl_test -rdc_dbg -rdc_dev -rdc_lib -rdc_run -rocdecode_dbg -rocdecode_dev -rocdecode_lib -rocgdb_dbg -rocgdb_dev -rocgdb_lib -rocgdb_run -rocjpeg_dbg -rocjpeg_dev -rocjpeg_lib -rocprofiler-compute_dbg -rocprofiler-compute_dev -rocprofiler-compute_lib -rocprofiler-compute_run -rocprofiler-sdk_dbg -rocprofiler-sdk_dev -rocprofiler-systems_dbg -rocprofiler-systems_dev -rocprofiler-systems_lib -rocprofiler-systems_run -rocr-debug-agent_dbg -rocr-debug-agent_dev -rocr-debug-agent_lib -rocrtst_dbg -rocrtst_dev -rocrtst_lib -rocrtst_run -rocwmma_dbg -rocwmma_dev -sysdeps_dev -sysdeps-amd-mesa_dev -sysdeps-amd-mesa_lib -sysdeps-expat_dev -sysdeps-gmp_dev -sysdeps-hwloc_dev -sysdeps-libpciaccess_dev -sysdeps-mpfr_dev -sysdeps-ncurses_dev diff --git a/docker/rocm_runtime_artifacts.txt b/docker/rocm_runtime_artifacts.txt deleted file mode 100644 index 94852ab4..00000000 --- a/docker/rocm_runtime_artifacts.txt +++ /dev/null @@ -1,56 +0,0 @@ -# ROCm runtime artifacts installed in the rocm_runtime stage. -# Lines starting with '#' are comments; blank lines are ignored. -# Each entry is an artifact prefix (_) consumed by -# fileset_tool.py artifact-flatten-split, which globs both the -# _generic shard and any per-arch shards produced by KPACK splitting. - -amd-llvm_lib -amd-llvm_run -# aqlprofile provides libhsa-amd-aqlprofile64.so, required at runtime by -# rocprofiler-sdk for PMC sampling and by anything loading rocprof -aqlprofile_lib -aqlprofile_run -base_lib -base_run -blas_lib -blas_run -composable-kernel_lib -core-amdsmi_lib -core-amdsmi_run -core-hip_lib -core-hip_run -core-kpack_lib -core-runtime_lib -core-runtime_run -fft_lib -fft_run -fftw3_lib -fftw3_run -hipblasltprovider_lib -hipdnn_lib -hipdnn_run -host-blas_lib -host-blas_run -host-suite-sparse_lib -host-suite-sparse_run -libhipcxx_lib -miopen_lib -miopen_run -miopenprovider_lib -rand_lib -rand_run -rccl_lib -rccl_run -# rocprofiler-sdk is needed at runtime: PyTorch Kineto (USE_KINETO=ON) depends -# on libroctracer64.so which in turn requires librocprofiler-sdk.so -rocprofiler-sdk_lib -rocprofiler-sdk_run -rocwmma_lib -sysdeps_lib -sysdeps_run -sysdeps-expat_lib -sysdeps-gmp_lib -sysdeps-hwloc_lib -sysdeps-libpciaccess_lib -sysdeps-mpfr_lib -sysdeps-ncurses_lib diff --git a/docker/setup_amdsmi.sh b/docker/setup_amdsmi.sh new file mode 100755 index 00000000..6a9c08b5 --- /dev/null +++ b/docker/setup_amdsmi.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Make `import amdsmi` work inside the venv. +# +# The amd-smi CLI lives at ${ROCM_HOME}/bin and is unaffected by this script. +# The nightly debs drop the Python bindings at ${ROCM_HOME}/share/amd_smi/amdsmi +# with no setup.py or pyproject.toml, so pip refuses to install them and they +# land nowhere on sys.path. torch.cuda wants them for device_count() and for +# utilization/memory_usage/temperature/power_draw/clock_rate; accelerate wants +# them for set_numa_affinity under ACCELERATE_CPU_AFFINITY. +# +# The fix is a symlink, never a copy. amdsmi_wrapper._load_library() finds the +# shared object through Path(__file__).resolve(), so a link resolves back into +# the ROCm tree and loads the matching lib/libamd_smi.so.; a copy +# breaks that and falls through to a bare-SONAME linker lookup. Putting +# share/amd_smi on sys.path instead (PYTHONPATH or a .pth file) is also wrong, +# because it exposes the sibling tools/, tests/ and example/ directories as +# implicit namespace packages. +# +# This shim is expected to expire. Once a nightly ships the bindings on sys.path +# on its own, the script changes nothing and prints a RETIRE banner instead -- +# the signal to delete it along with its COPY/RUN in docker/Dockerfile.ci. + +set -euo pipefail + +readonly rocm_root="${ROCM_HOME:-/opt/rocm}" +readonly src="${rocm_root}/share/amd_smi/amdsmi" + +site_packages="$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" +readonly site_packages +readonly link="${site_packages}/amdsmi" + +# Where does `import amdsmi` resolve from right now, if anywhere? +probe_origin() { + python3 -c \ + 'import importlib.util as u; print(getattr(u.find_spec("amdsmi"), "origin", "") or "")' \ + 2>/dev/null || true +} + +# Drop a link from an earlier run so the probe sees only what packaging +# provides. Anything else at that path came from elsewhere and is left alone. +if [[ -L "${link}" ]]; then + rm -f "${link}" +fi + +origin="$(probe_origin)" +if [[ -n "${origin}" ]]; then + echo "==============================================================" >&2 + echo " setup_amdsmi: RETIRE THIS SHIM" >&2 + echo " \`import amdsmi\` already resolves without it, from" >&2 + echo " ${origin}" >&2 + if [[ "${origin}" != "${rocm_root}/"* ]]; then + echo " Note: that path is outside ${rocm_root}, so the bindings are" >&2 + echo " probably a PyPI amdsmi wheel rather than a fixed nightly deb." >&2 + echo " Those bundle their own libamd_smi and can skew from ROCm." >&2 + fi + echo " No symlink was created. Delete docker/setup_amdsmi.sh and its" >&2 + echo " COPY/RUN in docker/Dockerfile.ci." >&2 + echo "==============================================================" >&2 + exit 0 +fi + +if [[ ! -d "${src}" ]]; then + echo "amdsmi is not importable and ${src} does not exist;" >&2 + echo "is the amdrocm-amdsmi deb installed in this layer?" >&2 + exit 1 +fi + +ln -sfT "${src}" "${link}" + +# Precompile in the ROCm tree so containers do not each rebuild __pycache__ on +# first import. Purely a nicety, so a failure here must not fail the build. +python3 -m compileall -q "${src}" > /dev/null || + echo "warning: could not precompile ${src}; imports compile at runtime" >&2 + +# The wrapper is import-tolerant: it swaps in a _MissingLibrary sentinel when +# the .so cannot be loaded, so a bare `import amdsmi` proves nothing. Check the +# library the wrapper actually resolved. +python3 - <<'PY' +import amdsmi +from amdsmi import amdsmi_wrapper + +missing = object() +lib = getattr(amdsmi_wrapper, "_loaded_lib_path", missing) +if lib is None: + raise SystemExit(f"linked {amdsmi.__file__} but libamd_smi.so failed to load") +if lib is missing: + print(f"linked {amdsmi.__file__} (library unverified: the wrapper no longer " + "exposes _loaded_lib_path)") +else: + print(f"linked {amdsmi.__file__} -> {lib}") +PY diff --git a/docker/setup_origami_cmake.sh b/docker/setup_origami_cmake.sh new file mode 100755 index 00000000..99de7a86 --- /dev/null +++ b/docker/setup_origami_cmake.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Provide a CMake package for origami so find_package(hipblaslt) can configure. +# +# hipBLASLt's installed hipblaslt-package-config.cmake does: +# find_dependency(origami CONFIG +# PATHS "${CMAKE_CURRENT_LIST_DIR}/../origami" +# "${CMAKE_CURRENT_LIST_DIR}/origami" +# NO_DEFAULT_PATH) +# CMAKE_PREFIX_PATH is ignored. Nightly debs currently ship liborigami.so in +# amdrocm-blas-host but omit lib/cmake/origami from amdrocm-blas-dev, so +# PyTorch's LoadHIP.cmake dies at find_package(hipblaslt REQUIRED). +# +# The fix is a one-file CMake config that imports the .so already on disk as +# roc::origami. It is not a copy of the upstream origami-config.cmake.in +# (that includes origami-targets.cmake and hip). +# +# This shim is expected to expire. Once a nightly ships origami-config.cmake +# (or origamiConfig.cmake) next to hipBLASLt on its own, the script changes +# nothing and prints a RETIRE banner instead -- the signal to delete it along +# with its COPY/RUN in docker/Dockerfile.ci. Independently, if hipBLASLt +# stops find_dependency(origami), this shim can be deleted even while the +# .so remains unpackaged as a CMake package; that case will not print the +# banner. + +set -euo pipefail + +readonly rocm_root="${ROCM_HOME:-${ROCM_PATH:-/opt/rocm}}" +readonly shim_marker="setup_origami_cmake: SHIM" +readonly dest_dir="${rocm_root}/lib/cmake/origami" +readonly dest="${dest_dir}/origami-config.cmake" + +# Paths hipBLASLt searches (NO_DEFAULT_PATH). First existing config wins. +config_candidates=( + "${rocm_root}/lib/cmake/origami/origami-config.cmake" + "${rocm_root}/lib/cmake/origami/origamiConfig.cmake" + "${rocm_root}/lib/cmake/hipblaslt/origami/origami-config.cmake" + "${rocm_root}/lib/cmake/hipblaslt/origami/origamiConfig.cmake" +) + +existing="" +for candidate in "${config_candidates[@]}"; do + if [[ -f "${candidate}" ]]; then + existing="${candidate}" + break + fi +done + +if [[ -n "${existing}" ]] && ! grep -q "${shim_marker}" "${existing}"; then + echo "==============================================================" >&2 + echo " setup_origami_cmake: RETIRE THIS SHIM" >&2 + echo " find_package(origami) already has a CMake config at" >&2 + echo " ${existing}" >&2 + echo " No stub was written. Delete docker/setup_origami_cmake.sh" >&2 + echo " and its COPY/RUN in docker/Dockerfile.ci." >&2 + echo "==============================================================" >&2 + exit 0 +fi + +lib="" +for candidate in \ + "${rocm_root}/lib/liborigami.so" \ + "${rocm_root}/lib/liborigami.so.1"; do + if [[ -e "${candidate}" ]]; then + lib="${candidate}" + break + fi +done + +if [[ -z "${lib}" ]]; then + echo "origami CMake config is missing and liborigami.so was not found" >&2 + echo "under ${rocm_root}/lib; is the amdrocm-blas-host deb installed" >&2 + echo "in this layer?" >&2 + exit 1 +fi + +mkdir -p "${dest_dir}" +cat > "${dest}" < $(readlink -f "${lib}")" diff --git a/docker/setup_rocm_llvm_layout.sh b/docker/setup_rocm_llvm_layout.sh new file mode 100755 index 00000000..ecfc9f4d --- /dev/null +++ b/docker/setup_rocm_llvm_layout.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Fill $ROCM_PATH/llvm/bin so FlyDSL/MLIR can find ld.lld. +# +# Nightly debs install the LLVM toolchain at ${ROCM_PATH}/lib/llvm/bin and leave +# ${ROCM_PATH}/llvm/bin as a stub of clang .cfg files. FlyDSL's bundled MLIR +# still serializes GPU binaries by invoking $ROCM_PATH/llvm/bin/ld.lld (classic +# toolkit layout), which fails with "lld invocation failed" when that path is +# missing. PATH lookups are a second, weaker contract; docker/Dockerfile.ci +# puts lib/llvm/bin on PATH via ENV and this script does not try to persist that. +# +# The fix is a symlink per missing name, never a copy, into the stub directory. +# Existing cfg files (and anything else already present) are left alone. +# +# This shim is expected to expire in either of two ways. Once a nightly ships +# an executable ld.lld under ${ROCM_PATH}/llvm/bin on its own, the script +# changes nothing and prints a RETIRE banner instead -- the signal to delete +# it along with its COPY/RUN in docker/Dockerfile.ci. Independently, if FlyDSL +# (or its bundled MLIR) stops looking at $ROCM_PATH/llvm/bin/ld.lld and uses +# lib/llvm/bin, HIP_CLANG_PATH, TRITON_HIP_LLD_PATH, or PATH instead, this +# shim can be deleted even while the stub directory remains; that case will +# not print the banner. PATH can stay: it is not this shim. + +set -euo pipefail + +readonly rocm_root="${ROCM_PATH:-${ROCM_HOME:-/opt/rocm}}" +readonly llvm_bin="${rocm_root}/lib/llvm/bin" +readonly classic_bin="${rocm_root}/llvm/bin" + +if [[ -x "${classic_bin}/ld.lld" ]]; then + echo "==============================================================" >&2 + echo " setup_rocm_llvm_layout: RETIRE THIS SHIM" >&2 + echo " ${classic_bin}/ld.lld is already executable:" >&2 + echo " $(readlink -f "${classic_bin}/ld.lld")" >&2 + echo " No extra tool symlinks were created. Delete" >&2 + echo " docker/setup_rocm_llvm_layout.sh and its COPY/RUN in" >&2 + echo " docker/Dockerfile.ci. Keep PATH=/opt/rocm/lib/llvm/bin in" >&2 + echo " the Dockerfile ENV unless a nightly also puts that on PATH." >&2 + echo "==============================================================" >&2 + exit 0 +fi + +if [[ ! -x "${llvm_bin}/ld.lld" ]]; then + echo "ld.lld is missing from both ${classic_bin} and ${llvm_bin};" >&2 + echo "is the amdrocm-core LLVM toolchain installed in this layer?" >&2 + exit 1 +fi + +mkdir -p "${classic_bin}" + +linked=0 +for src in "${llvm_bin}"/*; do + dest="${classic_bin}/$(basename "${src}")" + if [[ -e "${src}" && ! -e "${dest}" ]]; then + ln -s "${src}" "${dest}" + linked=$((linked + 1)) + fi +done + +if [[ ! -x "${classic_bin}/ld.lld" ]]; then + echo "linked ${linked} names from ${llvm_bin} into ${classic_bin}" >&2 + echo "but ${classic_bin}/ld.lld is still not executable" >&2 + exit 1 +fi + +echo "linked ${linked} LLVM toolchain names ${llvm_bin} -> ${classic_bin}" +echo "ld.lld -> $(readlink -f "${classic_bin}/ld.lld")" From 5e3d7a4342891877e6dcf7892068c0c74aece7fd Mon Sep 17 00:00:00 2001 From: Mikko Lauri Date: Wed, 30 Sep 2026 07:52:26 -0500 Subject: [PATCH 2/4] Document the nightly ROCm deb snapshot and shim retirement. Release notes now treat ROCM_RELEASE_ID and ROCM_DEB_SERIES as snapshot pins rather than TheRock commit overrides. Co-authored-by: Cursor --- .github/skills/create-release-notes/SKILL.md | 2 +- .../skills/create-release-notes/commands.md | 12 +-- CONTRIBUTING.md | 7 +- docker/README.md | 100 ++++++++++++++++++ 4 files changed, 113 insertions(+), 8 deletions(-) diff --git a/.github/skills/create-release-notes/SKILL.md b/.github/skills/create-release-notes/SKILL.md index 29b4c0c5..18d5ee38 100644 --- a/.github/skills/create-release-notes/SKILL.md +++ b/.github/skills/create-release-notes/SKILL.md @@ -23,7 +23,7 @@ Complete the shared [repository skill preflight](../preflight/SKILL.md) before c 2.2. [Resolve PRs associated with each commit](./commands.md#resolve-associated-prs) 2.3. If no public PR is associated, retain the commit SHA, subject, and changed files. Never convert a historical `#N` subject into a public PR link without API confirmation. - 2.4. [Diff dependency commit values](./commands.md#diff-dependency-commits) in `docker/Dockerfile.ci` + 2.4. [Diff dependency pins](./commands.md#diff-dependency-pins) in `docker/Dockerfile.ci` 2.5. For each changed commit hash, [look up upstream changes](./commands.md#look-up-upstream-changes) using the repo mapping 3. **Categorize each PR** by changed file paths per [categorization rules](./categorization.md) 3.1. Included (affect image): New Model, Performance, Docker/Environment, Benchmark Configs, Libraries diff --git a/.github/skills/create-release-notes/commands.md b/.github/skills/create-release-notes/commands.md index 7387d4e9..86ae3a69 100644 --- a/.github/skills/create-release-notes/commands.md +++ b/.github/skills/create-release-notes/commands.md @@ -46,20 +46,22 @@ For commits without an associated public PR, retain changed paths locally: git diff-tree --no-commit-id --name-only -r COMMIT_SHA ``` -## Diff dependency commits +## Diff dependency pins -Show which dependency commit values changed in `docker/Dockerfile.ci` between releases. -Include `*_COMMIT` pins and non-empty `*_COMMIT_OVERRIDE` values: +Show which dependency pins changed in `docker/Dockerfile.ci` between releases. +Include `*_COMMIT`, `ROCM_RELEASE_ID`, and `ROCM_DEB_SERIES`: ```bash git diff FROM_TAG..TO_REF -- docker/Dockerfile.ci | \ - grep -E '^[+-]ARG\s+\w+_COMMIT(_OVERRIDE)?=' | sort + grep -E '^[+-]ARG\s+(\w+_COMMIT|ROCM_RELEASE_ID|ROCM_DEB_SERIES)=' | sort ``` ## Look up upstream changes For each changed commit hash (`OLD_HASH` → `NEW_HASH`), query the upstream repo to see what was added. Use the repo mapping below to resolve the GitHub org/repo. +Record `ROCM_RELEASE_ID` and `ROCM_DEB_SERIES` changes as TheRock nightly deb +snapshot updates; they are not Git commit hashes. ```bash gh api repos/OWNER/REPO/compare/OLD_HASH...NEW_HASH \ @@ -83,8 +85,6 @@ gh api repos/OWNER/REPO/compare/OLD_HASH...NEW_HASH \ | XDIT_COMMIT | xdit-project/xDiT | | AO_COMMIT | pytorch/ao | | ARBITER_COMMIT | fal-ai/arbiter | -| ROCM_LIBRARIES_COMMIT_OVERRIDE | ROCm/rocm-libraries | -| ROCM_SYSTEMS_COMMIT_OVERRIDE | ROCm/rocm-systems | ## Release notes template diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b0b2511..998cc577 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -155,9 +155,14 @@ for various targets — see [docker/README.md](docker/README.md) for description To build an image locally: ```sh -docker build -f docker/Dockerfile.ci -t pytorch-xdit-dev . +docker build -f docker/Dockerfile.ci --target final -t pytorch-xdit-dev . ``` +This build does not require a GPU. The ROCm stack comes from the nightly deb +snapshot pinned by `ROCM_RELEASE_ID` and `ROCM_DEB_SERIES` in the Dockerfile. +See [docker/README.md](docker/README.md) for the ROCm package and architecture +settings. + The CI pipeline (`.github/workflows/build-and-benchmark.yml`) automates image builds and benchmark runs on supported hardware. The release path from staging to the public `rocm/pytorch-xdit` image is managed internally. diff --git a/docker/README.md b/docker/README.md index 5eb47b37..ab4951a7 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,3 +6,103 @@ - `Dockerfile.sgld_lmsys` SGL-D image based on `lmsysorg/sglang-rocm` with CI scripts from xDiT - `Dockerfile.vllm_omni` vLLM-Omni image for ROCm based on `amdsiloai/pytorch-xdit` - `Dockerfile.vllm_omni.cuda` vLLM-Omni image for CUDA based on `vllm/vllm-openai` + +## How-tos + +### ROCm nightly packages + +`Dockerfile.ci` installs ROCm from a pinned TheRock nightly deb snapshot on the +[legacy multi-arch native index](https://rocm.nightlies.amd.com/packages-multi-arch/deb/). +Update `ROCM_RELEASE_ID` and `ROCM_DEB_SERIES` together when moving to a new +snapshot or ROCm release series. `ROCM_GFX_TARGETS` controls which architecture-specific +package shards are installed and is independent of `PYTORCH_ROCM_ARCH`, which +controls the architectures built into PyTorch and related wheels. + +ROCm is layered as three named stages (later layers add packages only): + +| Target | Packages | Use | +| --- | --- | --- | +| `rocm_runtime` | `amdrocm-core${ROCM_DEB_SERIES}-${gfx}` | Runtime libs and `hipcc` | +| `rocm_runtime_jit` | plus `amdrocm-core-dev${ROCM_DEB_SERIES}-${gfx}` | Headers for AITER / Triton compile | +| `rocm_devel` | plus developer-tools, RDC, OpenCL, blas/rccl tests | Parent of `core` / `final` / `build_torch_stack` | + +`--target core` and `--target final` still inherit `rocm_devel` (full tools and +test debs). `rocm_runtime` and `rocm_runtime_jit` are ancestor cache layers and a +placeholder for a leaner product image; they are not CI tags. + +A later lean cutover is: point `core` at `rocm_runtime_jit`. That drops +developer-tools, RDC, OpenCL, and test debs from the shipped image, and the +`deps` rocprofiler-compute pip install must move or go away with that parent. +`build_torch_stack` can move to `rocm_runtime_jit` in the same change if torch +rebuilds should no longer follow test/tool deb churn. + +The image does not build ROCm from source and does not support +`rocm-libraries` or `rocm-systems` commit overrides. Changes that are not +available in the pinned nightly must first be published in a nightly snapshot. + +To build and validate every stage locally without a GPU: + +```sh +docker build -f docker/Dockerfile.ci --target final -t pytorch-xdit-dev . +``` + +### amd-smi Python bindings + +`docker/setup_amdsmi.sh` links the ROCm Python bindings into the venv, because +the nightly debs ship them under `${ROCM_HOME}/share/amd_smi/amdsmi` with no +`setup.py`, where pip cannot install them and nothing can import them. It runs +in `rocm_runtime`; the script header explains why a symlink is the only correct +mechanism. + +If a future nightly puts the bindings on `sys.path` itself, a build log prints +a `setup_amdsmi: RETIRE THIS SHIM` banner and the script makes no changes — +delete it and its `COPY`/`RUN` at that point. Watch for the banner when bumping `ROCM_RELEASE_ID`: + +```sh +docker build -f docker/Dockerfile.ci --target rocm_runtime . --progress=plain 2>&1 \ + | grep -i 'setup_amdsmi' +``` + +### LLVM classic toolkit path + +`docker/setup_rocm_llvm_layout.sh` links `${ROCM_PATH}/lib/llvm/bin` tools into +`${ROCM_PATH}/llvm/bin`. Nightly debs install `ld.lld` under `lib/llvm/bin` and +leave `llvm/bin` as clang `.cfg` stubs; FlyDSL/MLIR still invoke +`$ROCM_PATH/llvm/bin/ld.lld`. It runs in `rocm_runtime` after the ROCm `ENV` +block, which also puts `lib/llvm/bin` on `PATH`. + +If a future nightly ships an executable `ld.lld` at the classic path, a build +log prints a `setup_rocm_llvm_layout: RETIRE THIS SHIM` banner and the script +makes no changes — delete it and its `COPY`/`RUN` at that point. Keep the +`PATH=/opt/rocm/lib/llvm/bin` `ENV` unless the nightly also puts that directory +on `PATH`. The shim can also be deleted if FlyDSL (or its bundled MLIR) stops +looking for `$ROCM_PATH/llvm/bin/ld.lld` and uses `lib/llvm/bin`, +`HIP_CLANG_PATH`, `TRITON_HIP_LLD_PATH`, or `PATH` instead; that will not +print the banner. Watch for the banner when bumping `ROCM_RELEASE_ID`: + +```sh +docker build -f docker/Dockerfile.ci --target rocm_runtime . --progress=plain 2>&1 \ + | grep -i 'setup_rocm_llvm_layout' +``` + +### Origami CMake package + +`docker/setup_origami_cmake.sh` writes `${ROCM_HOME}/lib/cmake/origami/origami-config.cmake` +so `find_package(hipblaslt)` can configure. hipBLASLt's installed package config +requires `find_package(origami)` in a sibling `lib/cmake/origami` directory +(`NO_DEFAULT_PATH`, so `CMAKE_PREFIX_PATH` does not help). Nightly debs ship +`liborigami.so` in the BLAS host package but omit that CMake package from +BLAS devel. It runs in `rocm_runtime_jit` after the `-dev` debs, which is when +hipBLASLt's CMake files land. The script header explains why a stub imported +target is the only correct mechanism. + +If a future nightly ships `origami-config.cmake` (or `origamiConfig.cmake`) next +to hipBLASLt, a build log prints a `setup_origami_cmake: RETIRE THIS SHIM` +banner and the script makes no changes — delete it and its `COPY`/`RUN` at that +point. The shim can also be deleted if hipBLASLt stops `find_dependency(origami)`; +that will not print the banner. Watch for the banner when bumping `ROCM_RELEASE_ID`: + +```sh +docker build -f docker/Dockerfile.ci --target rocm_runtime_jit . --progress=plain 2>&1 \ + | grep -i 'setup_origami_cmake' +``` From 4ff92f4404b47fd39ad25b5a4d93d7ab1eee7ea4 Mon Sep 17 00:00:00 2001 From: Mikko Lauri Date: Mon, 5 Oct 2026 08:09:02 -0500 Subject: [PATCH 3/4] Record installed ROCm deb versions instead of TheRock build metadata. The nightly image has no /opt/rocm/share/therock tree, so both benchmark workflows record installed package versions from the image. The ITT workflow passes its runner list into the architecture matrix. Co-authored-by: Cursor --- .../actions/record-image-versions/action.yml | 29 ++++++ .../scripts/record-versions.sh | 90 +++++++++++++++++++ .github/workflows/benchmark-itt.yml | 14 +-- .github/workflows/build-and-benchmark.yml | 12 +-- 4 files changed, 127 insertions(+), 18 deletions(-) create mode 100644 .github/actions/record-image-versions/action.yml create mode 100755 .github/actions/record-image-versions/scripts/record-versions.sh diff --git a/.github/actions/record-image-versions/action.yml b/.github/actions/record-image-versions/action.yml new file mode 100644 index 00000000..4bcbc961 --- /dev/null +++ b/.github/actions/record-image-versions/action.yml @@ -0,0 +1,29 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +name: Record image versions +description: Record git HEAD and installed ROCm identity from a Docker image + +inputs: + image: + description: 'Docker image to inspect' + required: true + output_dir: + description: 'Directory for version artifacts' + required: false + default: 'versions' + container_name: + description: 'Temporary container name used for docker cp' + required: false + default: 'core' + +runs: + using: composite + steps: + - name: Record repository and ROCm versions + shell: bash + env: + IMAGE: ${{ inputs.image }} + OUTPUT_DIR: ${{ inputs.output_dir }} + CONTAINER_NAME: ${{ inputs.container_name }} + run: ${{ github.action_path }}/scripts/record-versions.sh diff --git a/.github/actions/record-image-versions/scripts/record-versions.sh b/.github/actions/record-image-versions/scripts/record-versions.sh new file mode 100755 index 00000000..b45f84ba --- /dev/null +++ b/.github/actions/record-image-versions/scripts/record-versions.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +set -euo pipefail + +# Copy ROCm identity out of IMAGE into OUTPUT_DIR using a created container. +# Expects env vars: IMAGE, OUTPUT_DIR, CONTAINER_NAME + +IMAGE="${IMAGE:?IMAGE is required}" +OUTPUT_DIR="${OUTPUT_DIR:?OUTPUT_DIR is required}" +CONTAINER_NAME="${CONTAINER_NAME:?CONTAINER_NAME is required}" + +mkdir -p "$OUTPUT_DIR" +git rev-parse HEAD > "$OUTPUT_DIR/repository.txt" + +docker pull "$IMAGE" +docker create --name "$CONTAINER_NAME" "$IMAGE" + +extract_dir=$(mktemp -d) +cleanup() { + docker rm -f "$CONTAINER_NAME" || true + rm -rf "$extract_dir" +} +trap cleanup EXIT + +mkdir -p \ + "$extract_dir/apt-sources" \ + "$extract_dir/rocm-info" \ + "$extract_dir/rocm-core-info" + +docker cp "${CONTAINER_NAME}:/var/lib/dpkg/status" "$extract_dir/dpkg-status" +docker cp "${CONTAINER_NAME}:/etc/apt/sources.list.d/." "$extract_dir/apt-sources/" || true +docker cp "${CONTAINER_NAME}:/opt/rocm/.info/." "$extract_dir/rocm-info/" || true +docker cp "${CONTAINER_NAME}:/opt/rocm/core/.info/." "$extract_dir/rocm-core-info/" || true +docker cp "${CONTAINER_NAME}:/opt/rocm/share/hip/version" "$extract_dir/hip-version" || true + +# TheRock debian versions encode the channel (stable X.Y.Z, +# nightly X.Y.Z~YYYYMMDD, prerelease X.Y.Z~preN, dev X.Y.Z~devYYYYMMDD). +# Classic ROCm apt uses rocm-* names; TheRock uses amdrocm-*. +packages="$(awk ' + function emit() { + if (pkg ~ /^(amdrocm|rocm)/ && status ~ /install ok installed/) + print pkg "\t" ver + pkg=""; ver=""; status="" + } + /^Package:/ { pkg=$2 } + /^Status:/ { status=$0 } + /^Version:/ { ver=$2 } + /^$/ { emit() } + END { emit() } +' "$extract_dir/dpkg-status")" +if [[ -z "${packages}" ]]; then + echo "(no amdrocm-* or rocm-* packages found)" >&2 + exit 1 +fi + +{ + echo "=== apt ROCm sources ===" + shopt -s nullglob + sources=("$extract_dir"/apt-sources/*rocm*) + if ((${#sources[@]})); then + cat "${sources[@]}" + else + echo "(none found)" + fi + echo + echo "=== installed ROCm packages ===" + echo "${packages}" + echo + echo "=== ROCm version files ===" + found= + version_files=( + "/opt/rocm/.info/version:$extract_dir/rocm-info/version" + "/opt/rocm/.info/version-dev:$extract_dir/rocm-info/version-dev" + "/opt/rocm/core/.info/version:$extract_dir/rocm-core-info/version" + "/opt/rocm/share/hip/version:$extract_dir/hip-version" + ) + for entry in "${version_files[@]}"; do + src="${entry%%:*}" + dest="${entry#*:}" + if [[ -r "$dest" ]]; then + echo "$src: $(cat "$dest")" + found=1 + fi + done + if [[ -z "${found}" ]]; then + echo "(none found)" + fi +} > "$OUTPUT_DIR/rocm.txt" diff --git a/.github/workflows/benchmark-itt.yml b/.github/workflows/benchmark-itt.yml index 4b1b2d63..57051fcc 100644 --- a/.github/workflows/benchmark-itt.yml +++ b/.github/workflows/benchmark-itt.yml @@ -114,16 +114,10 @@ jobs: benchmark_image: ${{ inputs.benchmark_image }} staging_image: ${{ env.STAGING_IMAGE }} - - name: Pull image and extract version information - env: - BENCHMARK_IMAGE: ${{ steps.resolve.outputs.image }} - run: | - mkdir -p versions - git rev-parse HEAD > versions/repository.txt - docker pull "$BENCHMARK_IMAGE" - docker create --name core "$BENCHMARK_IMAGE" - docker cp core:/opt/rocm/share/therock/ versions/ || true - docker rm core + - name: Record repository and ROCm versions + uses: ./.github/actions/record-image-versions + with: + image: ${{ steps.resolve.outputs.image }} - name: Upload version artifacts uses: actions/upload-artifact@v7 diff --git a/.github/workflows/build-and-benchmark.yml b/.github/workflows/build-and-benchmark.yml index c1986a64..f00cd0b5 100644 --- a/.github/workflows/build-and-benchmark.yml +++ b/.github/workflows/build-and-benchmark.yml @@ -294,14 +294,10 @@ jobs: with: run_number: ${{ github.run_number }} - - name: Pull image and extract version information - run: | - mkdir -p versions - git rev-parse HEAD > versions/repository.txt - docker pull ${{ steps.metadata.outputs.working_image }} - docker create --name core ${{ steps.metadata.outputs.working_image }} - docker cp core:/opt/rocm/share/therock/ versions/ || true - docker rm core + - name: Record repository and ROCm versions + uses: ./.github/actions/record-image-versions + with: + image: ${{ steps.metadata.outputs.working_image }} - name: Upload version artifacts uses: actions/upload-artifact@v7 From a025e215adeef8d8c150aca52b0a590e2448da5f Mon Sep 17 00:00:00 2001 From: Mikko Lauri Date: Mon, 5 Oct 2026 08:09:02 -0500 Subject: [PATCH 4/4] Install a lean ROCm runtime alongside the developer image. Replace the amdrocm-core metapackage with the libraries PyTorch and AITER JIT need. rocm_devel keeps LLVM static archives and developer tools; ROCM_PARENT=rocm_base builds the product stages without them. Co-authored-by: Cursor --- docker/Dockerfile.ci | 71 +++++++++++++++++------- docker/README.md | 51 +++++++++-------- docker/install_rocm_debs.sh | 107 +++++++++++++++++++++++++++++++----- 3 files changed, 170 insertions(+), 59 deletions(-) diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 74eb32ea..23c2c82e 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -17,6 +17,10 @@ ARG ROCM_RELEASE_ID="20260817-31980779979" ARG ROCM_DEB_SERIES="10.1" # ROCM_GFX_TARGETS selects architecture-specific nightly deb package shards. ARG ROCM_GFX_TARGETS="gfx942;gfx950" +# Parent of core, deps, and final. rocm_devel keeps LLVM static archives and +# adds developer tools. --build-arg ROCM_PARENT=rocm_base builds the same +# product stages without those archives or tools. +ARG ROCM_PARENT=rocm_devel # PYTORCH_ROCM_ARCH is a ";" separated list of values, choices for LLVM targets can # be found at https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html#supported-gpus ARG PYTORCH_ROCM_ARCH="gfx942;gfx950" @@ -173,49 +177,73 @@ RUN python3 -m venv venv ENV PATH="/venv/bin:$PATH" RUN pip install --no-cache-dir --upgrade pip packaging setuptools wheel -FROM base AS rocm_runtime +# rocm_tree and rocm_base install the same packages from `base`. The archive +# delete has to be inside the install it applies to: a child cannot drop +# files from a parent layer, and rocm_devel needs the archives this stage keeps. +FROM base AS rocm_tree ARG ROCM_RELEASE_ID ARG ROCM_DEB_SERIES ARG ROCM_GFX_TARGETS COPY docker/install_rocm_debs.sh /usr/local/bin/install_rocm_debs -RUN chmod +x /usr/local/bin/install_rocm_debs && install_rocm_debs runtime +COPY docker/setup_amdsmi.sh /usr/local/bin/setup_amdsmi +COPY docker/setup_rocm_llvm_layout.sh /usr/local/bin/setup_rocm_llvm_layout +COPY docker/setup_origami_cmake.sh /usr/local/bin/setup_origami_cmake +RUN chmod +x /usr/local/bin/install_rocm_debs \ + /usr/local/bin/setup_amdsmi \ + /usr/local/bin/setup_rocm_llvm_layout \ + /usr/local/bin/setup_origami_cmake \ + && install_rocm_debs base keep-archives # The nightly deb ships the amd-smi Python bindings somewhere pip cannot reach; # link them into site-packages. Prints a RETIRE banner and does nothing once a # nightly makes `import amdsmi` work on its own. -COPY docker/setup_amdsmi.sh /usr/local/bin/setup_amdsmi -RUN chmod +x /usr/local/bin/setup_amdsmi && setup_amdsmi +RUN setup_amdsmi ENV PATH="/opt/rocm/lib/llvm/bin:/opt/rocm/bin:$PATH" \ ROCM_HOME=/opt/rocm \ ROCM_PATH=/opt/rocm \ HIP_CLANG_PATH=/opt/rocm/lib/llvm/bin \ - HIP_DEVICE_LIB_PATH=/opt/rocm/lib/llvm/amdgcn/bitcode + HIP_DEVICE_LIB_PATH=/opt/rocm/lib/llvm/amdgcn/bitcode \ + TRITON_HIP_LLD_PATH=/opt/rocm/lib/llvm/bin/ld.lld # Nightly debs leave $ROCM_PATH/llvm/bin as a stub; FlyDSL/MLIR still # invoke ld.lld there. Prints a RETIRE banner and does nothing once a # nightly ships that path itself. Also retire if FlyDSL stops looking # there. PATH is the ENV above, not this shim. -COPY docker/setup_rocm_llvm_layout.sh /usr/local/bin/setup_rocm_llvm_layout -RUN chmod +x /usr/local/bin/setup_rocm_llvm_layout && setup_rocm_llvm_layout +RUN setup_rocm_llvm_layout +# hipBLASLt's CMake config find_dependency(origami) next to hipblaslt; +# nightly debs ship liborigami.so without lib/cmake/origami. Prints a +# RETIRE banner and does nothing once a nightly ships that package. +RUN setup_origami_cmake -FROM rocm_runtime AS rocm_runtime_jit +FROM base AS rocm_base ARG ROCM_RELEASE_ID ARG ROCM_DEB_SERIES ARG ROCM_GFX_TARGETS -RUN install_rocm_debs jit -# hipBLASLt's CMake config find_dependency(origami) next to hipblaslt; -# nightly debs ship liborigami.so without lib/cmake/origami. Prints a -# RETIRE banner and does nothing once a nightly ships that package. +COPY docker/install_rocm_debs.sh /usr/local/bin/install_rocm_debs +COPY docker/setup_amdsmi.sh /usr/local/bin/setup_amdsmi +COPY docker/setup_rocm_llvm_layout.sh /usr/local/bin/setup_rocm_llvm_layout COPY docker/setup_origami_cmake.sh /usr/local/bin/setup_origami_cmake -RUN chmod +x /usr/local/bin/setup_origami_cmake && setup_origami_cmake -ENV TRITON_HIP_LLD_PATH=/opt/rocm/lib/llvm/bin/ld.lld +RUN chmod +x /usr/local/bin/install_rocm_debs \ + /usr/local/bin/setup_amdsmi \ + /usr/local/bin/setup_rocm_llvm_layout \ + /usr/local/bin/setup_origami_cmake \ + && install_rocm_debs base strip-archives +RUN setup_amdsmi +ENV PATH="/opt/rocm/lib/llvm/bin:/opt/rocm/bin:$PATH" \ + ROCM_HOME=/opt/rocm \ + ROCM_PATH=/opt/rocm \ + HIP_CLANG_PATH=/opt/rocm/lib/llvm/bin \ + HIP_DEVICE_LIB_PATH=/opt/rocm/lib/llvm/amdgcn/bitcode \ + TRITON_HIP_LLD_PATH=/opt/rocm/lib/llvm/bin/ld.lld +RUN setup_rocm_llvm_layout +RUN setup_origami_cmake -FROM rocm_runtime_jit AS rocm_devel +FROM rocm_tree AS rocm_devel ARG ROCM_RELEASE_ID ARG ROCM_DEB_SERIES ARG ROCM_GFX_TARGETS RUN install_rocm_debs devel ENV PATH="/opt/rocm/libexec/rocprofiler-compute:$PATH" -FROM rocm_devel AS build_torch_stack +FROM rocm_tree AS build_torch_stack ARG THEROCK_COMMIT ARG TRITON_COMMIT ARG PYTORCH_COMMIT @@ -370,8 +398,7 @@ RUN python3 /usr/local/libexec/collect_image_licenses.py \ --output /ffmpeg/license-evidence \ --scan-root ffmpeg=${FFMPEG_SRC_DIR} -# Lean cutover: change this parent to rocm_runtime_jit. -FROM rocm_devel AS core +FROM ${ROCM_PARENT} AS core ARG AITER_COMMIT RUN --mount=type=bind,from=build_torch_stack,source=/install,target=/install \ pip install --no-cache-dir /install/*.whl @@ -486,10 +513,12 @@ RUN --mount=type=bind,source=patches,target=/patches \ git apply /patches/xfuser_sd3_joint_attn_kv_cache.patch && \ pip freeze | grep numpy > constraints.txt && \ pip install -c constraints.txt -e . -# rocprofiler-compute runtime dependencies +# rocprofiler-compute ships with rocm_devel. ROCM_PARENT=rocm_base has no tree. # Due to https://github.com/ROCm/rocm-systems/pull/2821 drop numpy to avoid installing numpy<2 -RUN sed -i '/numpy/d' ${ROCM_PATH}/libexec/rocprofiler-compute/requirements.txt && \ - pip install --no-cache-dir -r ${ROCM_PATH}/libexec/rocprofiler-compute/requirements.txt +RUN if [ -f "${ROCM_PATH}/libexec/rocprofiler-compute/requirements.txt" ]; then \ + sed -i '/numpy/d' "${ROCM_PATH}/libexec/rocprofiler-compute/requirements.txt" && \ + pip install --no-cache-dir -r "${ROCM_PATH}/libexec/rocprofiler-compute/requirements.txt"; \ + fi # imageio + imageio-ffmpeg for diffusers export_to_video H.264 encoding # --no-binary avoids shipping imageio-ffmpeg's bundled GPL ffmpeg binary RUN pip install --no-cache-dir --no-binary :all: imageio-ffmpeg diff --git a/docker/README.md b/docker/README.md index ab4951a7..098caf9d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -18,40 +18,43 @@ snapshot or ROCm release series. `ROCM_GFX_TARGETS` controls which architecture- package shards are installed and is independent of `PYTORCH_ROCM_ARCH`, which controls the architectures built into PyTorch and related wheels. -ROCm is layered as three named stages (later layers add packages only): +Leaf packages install under `/opt/rocm/core-`. `rocm_tree` and `rocm_base` +both register the unversioned `/opt/rocm/{bin,lib,include,...}` links that the +`amdrocm-core` postinst would have created. They are separate installs from +`base`: `rocm_base` drops LLVM static archives outside `lib/clang` in that same +install step, and `rocm_tree` keeps them so `rocm_devel` can inherit the files. | Target | Packages | Use | | --- | --- | --- | -| `rocm_runtime` | `amdrocm-core${ROCM_DEB_SERIES}-${gfx}` | Runtime libs and `hipcc` | -| `rocm_runtime_jit` | plus `amdrocm-core-dev${ROCM_DEB_SERIES}-${gfx}` | Headers for AITER / Triton compile | -| `rocm_devel` | plus developer-tools, RDC, OpenCL, blas/rccl tests | Parent of `core` / `final` / `build_torch_stack` | +| `rocm_tree` | HIP/`hipcc`, BLAS, MIOpen, RCCL, FFT, RAND, sparse, and solver for each arch in `ROCM_GFX_TARGETS`, plus headers. LLVM static archives kept | Parent of `build_torch_stack` and `rocm_devel` | +| `rocm_base` | Same packages. LLVM static archives outside `lib/clang` removed | Lean runtime target | +| `rocm_devel` | `rocm_tree` plus developer-tools, RDC, OpenCL, blas/rccl tests | Default parent of `core` / `deps` / `final` | -`--target core` and `--target final` still inherit `rocm_devel` (full tools and -test debs). `rocm_runtime` and `rocm_runtime_jit` are ancestor cache layers and a -placeholder for a leaner product image; they are not CI tags. +`amdrocm-developer-tools` pulls the profiler, emulator, and debugger. It does +not depend on `amdrocm-core`. `core` is `FROM ${ROCM_PARENT}`, which defaults to +`rocm_devel`. Both parents install the wheels built by `build_torch_stack`. +`deps` installs rocprofiler-compute's Python requirements only when that tree +is present. -A later lean cutover is: point `core` at `rocm_runtime_jit`. That drops -developer-tools, RDC, OpenCL, and test debs from the shipped image, and the -`deps` rocprofiler-compute pip install must move or go away with that parent. -`build_torch_stack` can move to `rocm_runtime_jit` in the same change if torch -rebuilds should no longer follow test/tool deb churn. +```sh +# developer tools included +docker build -f docker/Dockerfile.ci --target final -t pytorch-xdit-dev . + +# runtime + torch stack + AITER JIT, without developer tools +docker build -f docker/Dockerfile.ci --target final \ + --build-arg ROCM_PARENT=rocm_base -t pytorch-xdit-lean . +``` The image does not build ROCm from source and does not support `rocm-libraries` or `rocm-systems` commit overrides. Changes that are not available in the pinned nightly must first be published in a nightly snapshot. -To build and validate every stage locally without a GPU: - -```sh -docker build -f docker/Dockerfile.ci --target final -t pytorch-xdit-dev . -``` - ### amd-smi Python bindings `docker/setup_amdsmi.sh` links the ROCm Python bindings into the venv, because the nightly debs ship them under `${ROCM_HOME}/share/amd_smi/amdsmi` with no `setup.py`, where pip cannot install them and nothing can import them. It runs -in `rocm_runtime`; the script header explains why a symlink is the only correct +in `rocm_tree` and `rocm_base`; the script header explains why a symlink is the only correct mechanism. If a future nightly puts the bindings on `sys.path` itself, a build log prints @@ -59,7 +62,7 @@ a `setup_amdsmi: RETIRE THIS SHIM` banner and the script makes no changes — delete it and its `COPY`/`RUN` at that point. Watch for the banner when bumping `ROCM_RELEASE_ID`: ```sh -docker build -f docker/Dockerfile.ci --target rocm_runtime . --progress=plain 2>&1 \ +docker build -f docker/Dockerfile.ci --target rocm_base . --progress=plain 2>&1 \ | grep -i 'setup_amdsmi' ``` @@ -68,7 +71,7 @@ docker build -f docker/Dockerfile.ci --target rocm_runtime . --progress=plain 2> `docker/setup_rocm_llvm_layout.sh` links `${ROCM_PATH}/lib/llvm/bin` tools into `${ROCM_PATH}/llvm/bin`. Nightly debs install `ld.lld` under `lib/llvm/bin` and leave `llvm/bin` as clang `.cfg` stubs; FlyDSL/MLIR still invoke -`$ROCM_PATH/llvm/bin/ld.lld`. It runs in `rocm_runtime` after the ROCm `ENV` +`$ROCM_PATH/llvm/bin/ld.lld`. It runs in `rocm_tree` and `rocm_base` after the ROCm `ENV` block, which also puts `lib/llvm/bin` on `PATH`. If a future nightly ships an executable `ld.lld` at the classic path, a build @@ -81,7 +84,7 @@ looking for `$ROCM_PATH/llvm/bin/ld.lld` and uses `lib/llvm/bin`, print the banner. Watch for the banner when bumping `ROCM_RELEASE_ID`: ```sh -docker build -f docker/Dockerfile.ci --target rocm_runtime . --progress=plain 2>&1 \ +docker build -f docker/Dockerfile.ci --target rocm_base . --progress=plain 2>&1 \ | grep -i 'setup_rocm_llvm_layout' ``` @@ -92,7 +95,7 @@ so `find_package(hipblaslt)` can configure. hipBLASLt's installed package config requires `find_package(origami)` in a sibling `lib/cmake/origami` directory (`NO_DEFAULT_PATH`, so `CMAKE_PREFIX_PATH` does not help). Nightly debs ship `liborigami.so` in the BLAS host package but omit that CMake package from -BLAS devel. It runs in `rocm_runtime_jit` after the `-dev` debs, which is when +BLAS devel. It runs in `rocm_tree` and `rocm_base` after the `-dev` debs, which is when hipBLASLt's CMake files land. The script header explains why a stub imported target is the only correct mechanism. @@ -103,6 +106,6 @@ point. The shim can also be deleted if hipBLASLt stops `find_dependency(origami) that will not print the banner. Watch for the banner when bumping `ROCM_RELEASE_ID`: ```sh -docker build -f docker/Dockerfile.ci --target rocm_runtime_jit . --progress=plain 2>&1 \ +docker build -f docker/Dockerfile.ci --target rocm_base . --progress=plain 2>&1 \ | grep -i 'setup_origami_cmake' ``` diff --git a/docker/install_rocm_debs.sh b/docker/install_rocm_debs.sh index 446a0f73..5a69b41e 100755 --- a/docker/install_rocm_debs.sh +++ b/docker/install_rocm_debs.sh @@ -4,17 +4,33 @@ set -euo pipefail -readonly layer="${1:?usage: install_rocm_debs runtime|jit|devel}" +readonly layer="${1:?usage: install_rocm_debs base strip-archives|keep-archives | devel}" +readonly archives="${2:-}" readonly release_id="${ROCM_RELEASE_ID:?ROCM_RELEASE_ID must be set}" readonly deb_series="${ROCM_DEB_SERIES:?ROCM_DEB_SERIES must be set}" readonly gfx_targets="${ROCM_GFX_TARGETS:?ROCM_GFX_TARGETS must be set}" readonly rocm_root="${ROCM_HOME:-/opt/rocm}" +readonly prefix="/opt/rocm/core-${deb_series}" readonly repo_url="https://rocm.nightlies.amd.com/packages-multi-arch/deb/${release_id}" case "${layer}" in - runtime|jit|devel) ;; + base) + case "${archives}" in + strip-archives|keep-archives) ;; + *) + echo "base layer requires strip-archives or keep-archives" >&2 + exit 2 + ;; + esac + ;; + devel) + if [[ -n "${archives}" ]]; then + echo "devel layer takes no archive mode" >&2 + exit 2 + fi + ;; *) - echo "unknown layer '${layer}'; expected runtime, jit, or devel" >&2 + echo "unknown layer '${layer}'; expected base or devel" >&2 exit 2 ;; esac @@ -30,19 +46,43 @@ fi packages=() case "${layer}" in - runtime) - for target in "${targets[@]}"; do - [[ -n "${target}" ]] || continue - packages+=("amdrocm-core${deb_series}-${target}") - done - ;; - jit) + base) + # Compiler, HIP runtime, and headers. runtime-dev depends on llvm-dev. + # Host libraries the AITER JIT link line names, plus the runtimes + # libtorch_hip loads. Per-arch shards hold device code. solver/sparse + # gfx packages are empty edges pulled in by the BLAS shard. + # rocSHMEM, hipFile, rocJPEG, rocDecode, and RPP stay out: this + # libtorch does not link them, and amdrocm-core would pull them in. + packages+=( + "amdrocm-runtime-dev${deb_series}" + "amdrocm-ccl-dev${deb_series}" + "amdrocm-blas-dev${deb_series}" + "amdrocm-blas-host${deb_series}" + "amdrocm-hipblas-common-dev${deb_series}" + "amdrocm-solver-dev${deb_series}" + "amdrocm-solver-host${deb_series}" + "amdrocm-dnn-dev${deb_series}" + "amdrocm-fft-dev${deb_series}" + "amdrocm-rand-dev${deb_series}" + "amdrocm-sparse-dev${deb_series}" + "amdrocm-rccl-dev${deb_series}" + "amdrocm-amdsmi${deb_series}" + ) for target in "${targets[@]}"; do [[ -n "${target}" ]] || continue - packages+=("amdrocm-core-dev${deb_series}-${target}") + packages+=( + "amdrocm-blas${deb_series}-${target}" + "amdrocm-dnn${deb_series}-${target}" + "amdrocm-rccl${deb_series}-${target}" + "amdrocm-rand${deb_series}-${target}" + "amdrocm-fft${deb_series}-${target}" + ) done ;; devel) + # developer-tools depends on the profiler, emulator, and debugger. + # It does not depend on amdrocm-core, so this layer does not pull + # the libraries rocm_base left out. packages=( "amdrocm-developer-tools${deb_series}" "amdrocm-rdc${deb_series}" @@ -67,6 +107,26 @@ apt-get update -qq apt-get install -y --no-install-recommends "${packages[@]}" rm -rf /var/lib/apt/lists/* +# Leaf packages install under /opt/rocm/core-. The unversioned +# /opt/rocm/{bin,lib,include,...} links normally come from the amdrocm-core +# postinst, which this image does not install. +if [[ "${layer}" == "base" ]]; then + test -x "${prefix}/bin/hipcc" + test -e "${prefix}/lib/libamdhip64.so" + test -d "${prefix}/lib/llvm/amdgcn/bitcode" + major="${deb_series%%.*}" + score=100 + update-alternatives --install /opt/rocm/core core "${prefix}" "${score}" + update-alternatives --install "/opt/rocm/core-${major}" "core-${major}" "${prefix}" "${score}" + update-alternatives --install /opt/rocm/bin rocm-bin "${prefix}/bin" "${score}" + update-alternatives --install /opt/rocm/lib rocm-lib "${prefix}/lib" "${score}" + update-alternatives --install /opt/rocm/include rocm-include "${prefix}/include" "${score}" + update-alternatives --install /opt/rocm/llvm rocm-llvm "${prefix}/llvm" "${score}" + update-alternatives --install /opt/rocm/amdgcn rocm-amdgcn "${prefix}/amdgcn" "${score}" + update-alternatives --install /opt/rocm/libexec rocm-libexec "${prefix}/libexec" "${score}" + update-alternatives --install /opt/rocm/share rocm-share "${prefix}/share" "${score}" +fi + echo "${rocm_root}/lib" > /etc/ld.so.conf.d/rocm.conf if [[ -d "${rocm_root}/lib/rocm_sysdeps/lib" ]]; then echo "${rocm_root}/lib/rocm_sysdeps/lib" \ @@ -74,21 +134,40 @@ if [[ -d "${rocm_root}/lib/rocm_sysdeps/lib" ]]; then fi ldconfig +# hipcc links libLLVM.so / libclang-cpp.so. llvm-dev is a hard dependency of +# runtime-dev and is mostly static archives, including the Flang frontend. +# AITER JIT and the torch link do not static-link LLVM. The host link still +# needs libclang_rt.builtins.a under lib/clang/, so that tree stays. +# strip-archives deletes in this same step as the install. A later layer +# would leave the archives in the image. keep-archives is the rocm_tree +# install that rocm_devel inherits. The package stays installed either way. +if [[ "${archives}" == "strip-archives" ]]; then + find "${prefix}/lib/llvm/lib" -name '*.a' -not -path '*/lib/clang/*' -delete +fi + case "${layer}" in - runtime) + base) test -d "${rocm_root}/lib/llvm/amdgcn/bitcode" test -e "${rocm_root}/lib/llvm/bin/clang++" test -e "${rocm_root}/lib/libamdhip64.so" test -d "${rocm_root}/lib/rocm_sysdeps/lib" test -e "${rocm_root}/lib/librocprofiler-sdk.so" test -e "${rocm_root}/lib/libroctracer64.so" - ;; - jit) test -d "${rocm_root}/include/roctracer" test -d "${rocm_root}/lib/rocm_sysdeps/include" test -d "${rocm_root}/lib/rocm_sysdeps/lib/pkgconfig" + test -e "${rocm_root}/lib/libMIOpen.so" + test -e "${rocm_root}/lib/librccl.so" + test -e "${rocm_root}/lib/libhipblaslt.so" + frontend="${prefix}/lib/llvm/lib/libflangFrontend.a" + if [[ "${archives}" == "keep-archives" ]]; then + test -e "${frontend}" + else + test ! -e "${frontend}" + fi ;; devel) test -d "${rocm_root}/libexec/rocprofiler-compute" + test -e "${prefix}/lib/llvm/lib/libflangFrontend.a" ;; esac