diff --git a/.github/actions/record-image-versions/action.yml b/.github/actions/record-image-versions/action.yml new file mode 100644 index 00000000..4bcbc961 --- /dev/null +++ b/.github/actions/record-image-versions/action.yml @@ -0,0 +1,29 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +name: Record image versions +description: Record git HEAD and installed ROCm identity from a Docker image + +inputs: + image: + description: 'Docker image to inspect' + required: true + output_dir: + description: 'Directory for version artifacts' + required: false + default: 'versions' + container_name: + description: 'Temporary container name used for docker cp' + required: false + default: 'core' + +runs: + using: composite + steps: + - name: Record repository and ROCm versions + shell: bash + env: + IMAGE: ${{ inputs.image }} + OUTPUT_DIR: ${{ inputs.output_dir }} + CONTAINER_NAME: ${{ inputs.container_name }} + run: ${{ github.action_path }}/scripts/record-versions.sh diff --git a/.github/actions/record-image-versions/scripts/record-versions.sh b/.github/actions/record-image-versions/scripts/record-versions.sh new file mode 100755 index 00000000..b45f84ba --- /dev/null +++ b/.github/actions/record-image-versions/scripts/record-versions.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +set -euo pipefail + +# Copy ROCm identity out of IMAGE into OUTPUT_DIR using a created container. +# Expects env vars: IMAGE, OUTPUT_DIR, CONTAINER_NAME + +IMAGE="${IMAGE:?IMAGE is required}" +OUTPUT_DIR="${OUTPUT_DIR:?OUTPUT_DIR is required}" +CONTAINER_NAME="${CONTAINER_NAME:?CONTAINER_NAME is required}" + +mkdir -p "$OUTPUT_DIR" +git rev-parse HEAD > "$OUTPUT_DIR/repository.txt" + +docker pull "$IMAGE" +docker create --name "$CONTAINER_NAME" "$IMAGE" + +extract_dir=$(mktemp -d) +cleanup() { + docker rm -f "$CONTAINER_NAME" || true + rm -rf "$extract_dir" +} +trap cleanup EXIT + +mkdir -p \ + "$extract_dir/apt-sources" \ + "$extract_dir/rocm-info" \ + "$extract_dir/rocm-core-info" + +docker cp "${CONTAINER_NAME}:/var/lib/dpkg/status" "$extract_dir/dpkg-status" +docker cp "${CONTAINER_NAME}:/etc/apt/sources.list.d/." "$extract_dir/apt-sources/" || true +docker cp "${CONTAINER_NAME}:/opt/rocm/.info/." "$extract_dir/rocm-info/" || true +docker cp "${CONTAINER_NAME}:/opt/rocm/core/.info/." "$extract_dir/rocm-core-info/" || true +docker cp "${CONTAINER_NAME}:/opt/rocm/share/hip/version" "$extract_dir/hip-version" || true + +# TheRock debian versions encode the channel (stable X.Y.Z, +# nightly X.Y.Z~YYYYMMDD, prerelease X.Y.Z~preN, dev X.Y.Z~devYYYYMMDD). +# Classic ROCm apt uses rocm-* names; TheRock uses amdrocm-*. +packages="$(awk ' + function emit() { + if (pkg ~ /^(amdrocm|rocm)/ && status ~ /install ok installed/) + print pkg "\t" ver + pkg=""; ver=""; status="" + } + /^Package:/ { pkg=$2 } + /^Status:/ { status=$0 } + /^Version:/ { ver=$2 } + /^$/ { emit() } + END { emit() } +' "$extract_dir/dpkg-status")" +if [[ -z "${packages}" ]]; then + echo "(no amdrocm-* or rocm-* packages found)" >&2 + exit 1 +fi + +{ + echo "=== apt ROCm sources ===" + shopt -s nullglob + sources=("$extract_dir"/apt-sources/*rocm*) + if ((${#sources[@]})); then + cat "${sources[@]}" + else + echo "(none found)" + fi + echo + echo "=== installed ROCm packages ===" + echo "${packages}" + echo + echo "=== ROCm version files ===" + found= + version_files=( + "/opt/rocm/.info/version:$extract_dir/rocm-info/version" + "/opt/rocm/.info/version-dev:$extract_dir/rocm-info/version-dev" + "/opt/rocm/core/.info/version:$extract_dir/rocm-core-info/version" + "/opt/rocm/share/hip/version:$extract_dir/hip-version" + ) + for entry in "${version_files[@]}"; do + src="${entry%%:*}" + dest="${entry#*:}" + if [[ -r "$dest" ]]; then + echo "$src: $(cat "$dest")" + found=1 + fi + done + if [[ -z "${found}" ]]; then + echo "(none found)" + fi +} > "$OUTPUT_DIR/rocm.txt" diff --git a/.github/skills/create-release-notes/SKILL.md b/.github/skills/create-release-notes/SKILL.md index 29b4c0c5..18d5ee38 100644 --- a/.github/skills/create-release-notes/SKILL.md +++ b/.github/skills/create-release-notes/SKILL.md @@ -23,7 +23,7 @@ Complete the shared [repository skill preflight](../preflight/SKILL.md) before c 2.2. [Resolve PRs associated with each commit](./commands.md#resolve-associated-prs) 2.3. If no public PR is associated, retain the commit SHA, subject, and changed files. Never convert a historical `#N` subject into a public PR link without API confirmation. - 2.4. [Diff dependency commit values](./commands.md#diff-dependency-commits) in `docker/Dockerfile.ci` + 2.4. [Diff dependency pins](./commands.md#diff-dependency-pins) in `docker/Dockerfile.ci` 2.5. For each changed commit hash, [look up upstream changes](./commands.md#look-up-upstream-changes) using the repo mapping 3. **Categorize each PR** by changed file paths per [categorization rules](./categorization.md) 3.1. Included (affect image): New Model, Performance, Docker/Environment, Benchmark Configs, Libraries diff --git a/.github/skills/create-release-notes/commands.md b/.github/skills/create-release-notes/commands.md index 7387d4e9..86ae3a69 100644 --- a/.github/skills/create-release-notes/commands.md +++ b/.github/skills/create-release-notes/commands.md @@ -46,20 +46,22 @@ For commits without an associated public PR, retain changed paths locally: git diff-tree --no-commit-id --name-only -r COMMIT_SHA ``` -## Diff dependency commits +## Diff dependency pins -Show which dependency commit values changed in `docker/Dockerfile.ci` between releases. -Include `*_COMMIT` pins and non-empty `*_COMMIT_OVERRIDE` values: +Show which dependency pins changed in `docker/Dockerfile.ci` between releases. +Include `*_COMMIT`, `ROCM_RELEASE_ID`, and `ROCM_DEB_SERIES`: ```bash git diff FROM_TAG..TO_REF -- docker/Dockerfile.ci | \ - grep -E '^[+-]ARG\s+\w+_COMMIT(_OVERRIDE)?=' | sort + grep -E '^[+-]ARG\s+(\w+_COMMIT|ROCM_RELEASE_ID|ROCM_DEB_SERIES)=' | sort ``` ## Look up upstream changes For each changed commit hash (`OLD_HASH` → `NEW_HASH`), query the upstream repo to see what was added. Use the repo mapping below to resolve the GitHub org/repo. +Record `ROCM_RELEASE_ID` and `ROCM_DEB_SERIES` changes as TheRock nightly deb +snapshot updates; they are not Git commit hashes. ```bash gh api repos/OWNER/REPO/compare/OLD_HASH...NEW_HASH \ @@ -83,8 +85,6 @@ gh api repos/OWNER/REPO/compare/OLD_HASH...NEW_HASH \ | XDIT_COMMIT | xdit-project/xDiT | | AO_COMMIT | pytorch/ao | | ARBITER_COMMIT | fal-ai/arbiter | -| ROCM_LIBRARIES_COMMIT_OVERRIDE | ROCm/rocm-libraries | -| ROCM_SYSTEMS_COMMIT_OVERRIDE | ROCm/rocm-systems | ## Release notes template diff --git a/.github/workflows/benchmark-itt.yml b/.github/workflows/benchmark-itt.yml index 4b1b2d63..57051fcc 100644 --- a/.github/workflows/benchmark-itt.yml +++ b/.github/workflows/benchmark-itt.yml @@ -114,16 +114,10 @@ jobs: benchmark_image: ${{ inputs.benchmark_image }} staging_image: ${{ env.STAGING_IMAGE }} - - name: Pull image and extract version information - env: - BENCHMARK_IMAGE: ${{ steps.resolve.outputs.image }} - run: | - mkdir -p versions - git rev-parse HEAD > versions/repository.txt - docker pull "$BENCHMARK_IMAGE" - docker create --name core "$BENCHMARK_IMAGE" - docker cp core:/opt/rocm/share/therock/ versions/ || true - docker rm core + - name: Record repository and ROCm versions + uses: ./.github/actions/record-image-versions + with: + image: ${{ steps.resolve.outputs.image }} - name: Upload version artifacts uses: actions/upload-artifact@v7 diff --git a/.github/workflows/build-and-benchmark.yml b/.github/workflows/build-and-benchmark.yml index c1986a64..f00cd0b5 100644 --- a/.github/workflows/build-and-benchmark.yml +++ b/.github/workflows/build-and-benchmark.yml @@ -294,14 +294,10 @@ jobs: with: run_number: ${{ github.run_number }} - - name: Pull image and extract version information - run: | - mkdir -p versions - git rev-parse HEAD > versions/repository.txt - docker pull ${{ steps.metadata.outputs.working_image }} - docker create --name core ${{ steps.metadata.outputs.working_image }} - docker cp core:/opt/rocm/share/therock/ versions/ || true - docker rm core + - name: Record repository and ROCm versions + uses: ./.github/actions/record-image-versions + with: + image: ${{ steps.metadata.outputs.working_image }} - name: Upload version artifacts uses: actions/upload-artifact@v7 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b0b2511..998cc577 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -155,9 +155,14 @@ for various targets — see [docker/README.md](docker/README.md) for description To build an image locally: ```sh -docker build -f docker/Dockerfile.ci -t pytorch-xdit-dev . +docker build -f docker/Dockerfile.ci --target final -t pytorch-xdit-dev . ``` +This build does not require a GPU. The ROCm stack comes from the nightly deb +snapshot pinned by `ROCM_RELEASE_ID` and `ROCM_DEB_SERIES` in the Dockerfile. +See [docker/README.md](docker/README.md) for the ROCm package and architecture +settings. + The CI pipeline (`.github/workflows/build-and-benchmark.yml`) automates image builds and benchmark runs on supported hardware. The release path from staging to the public `rocm/pytorch-xdit` image is managed internally. diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index d76258c1..23c2c82e 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -13,24 +13,17 @@ ARG BASE_IMAGE="ubuntu:24.04" ARG PYTHON_VERSION="default" ARG THEROCK_COMMIT="ab56f1f7c5b54746bf9f7948fff3f306b2bef8b2" -ARG CCACHE_VERSION="4.11.2" -# AMDGPU_FAMILIES is a ";" separated list of values, choices can be found -# at https://github.com/ROCm/TheRock/blob/main/cmake/therock_amdgpu_targets.cmake -ARG AMDGPU_FAMILIES="gfx94X-dcgpu;gfx950-dcgpu" -# AMDGPU_BUNDLE_NAME disambiguates the cmake-level dist bundle name when more -# than one AMDGPU family is selected. TheRock's therock_validate_amdgpu_targets() -# FATAL_ERRORs if the bundle name cannot be inferred from a single family. -# With KPACK_SPLIT_ARTIFACTS=ON it only appears in the intermediate -# build/artifacts-unsplit/ tree and is not observable in the final image. -ARG AMDGPU_BUNDLE_NAME="gfx94X_gfx950_bundle" +ARG ROCM_RELEASE_ID="20260817-31980779979" +ARG ROCM_DEB_SERIES="10.1" +# ROCM_GFX_TARGETS selects architecture-specific nightly deb package shards. +ARG ROCM_GFX_TARGETS="gfx942;gfx950" +# Parent of core, deps, and final. rocm_devel keeps LLVM static archives and +# adds developer tools. --build-arg ROCM_PARENT=rocm_base builds the same +# product stages without those archives or tools. +ARG ROCM_PARENT=rocm_devel # PYTORCH_ROCM_ARCH is a ";" separated list of values, choices for LLVM targets can # be found at https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html#supported-gpus ARG PYTORCH_ROCM_ARCH="gfx942;gfx950" -# Set ROCM_LIBRARIES_COMMIT_OVERRIDE to force to use a specific commit of rocm-libraries monorepo instead of the one pinned in TheRock. -# This is useful for example for testing a feature not yet in the main branch of rocm-libraries or a commit not available via TheRock. -ARG ROCM_LIBRARIES_COMMIT_OVERRIDE="" -# Set ROCM_SYSTEMS_COMMIT_OVERRIDE to force to use a specific commit of rocm-systems monorepo instead of the one pinned in TheRock. -ARG ROCM_SYSTEMS_COMMIT_OVERRIDE="" ARG TRITON_COMMIT="669b31acc1dd1b3fd93286afd5db67f65d9f7557" ARG PYTORCH_COMMIT="3369fa23f831832166651a0b8e151dc2877f3c77" ARG PYTORCH_VISION_COMMIT="53881ee74892a7c22f2b16a84b15a00f64a8bf68" @@ -110,39 +103,16 @@ FROM base_os AS base ARG PYTHON_VERSION ARG DEBIAN_FRONTEND=noninteractive -# install depedencies +# Shared fetch/compiler tools. Native extensions in core (AITER, torchao) +# and the torch/ffmpeg builders still need a host toolchain. RUN apt-get update -qq && apt-get install -y --no-install-recommends \ - automake \ - bison \ build-essential \ ca-certificates \ cmake \ curl \ - flex \ - freeglut3-dev \ - gfortran \ git \ - libgfortran5 \ - libgl1 \ - libgl1-mesa-dev \ - libglib2.0-dev \ - libglu1-mesa-dev \ - libopenblas-dev \ - # Workaround: rocprofiler-sdk SQL output sources include , but - # rocm-systems#6664 no longer propagates bundled SQLite headers to - # rocprofiler-sdk-output-library. Remove once fixed upstream: - # https://github.com/ROCm/TheRock/issues/5868 - libsqlite3-dev \ - # Workaround: profiler-hub generates its bundled SQLite3 amalgamation via - # `make sqlite3.c`, which requires tclsh even with --disable-tcl. Remove - # once fixed upstream: - # https://github.com/ROCm/rocm-systems/issues/10059 - tcl \ - libtool \ ninja-build \ - texinfo \ - wget \ - xxd && \ + wget && \ update-ca-certificates && \ rm -rf /var/lib/apt/lists/* @@ -207,120 +177,95 @@ RUN python3 -m venv venv ENV PATH="/venv/bin:$PATH" RUN pip install --no-cache-dir --upgrade pip packaging setuptools wheel -FROM base AS setup_therock -ARG THEROCK_COMMIT -ARG ROCM_LIBRARIES_COMMIT_OVERRIDE -ARG ROCM_SYSTEMS_COMMIT_OVERRIDE - -WORKDIR /app -RUN git clone https://github.com/ROCm/TheRock.git && \ - cd TheRock && \ - git reset --hard ${THEROCK_COMMIT} && \ - if [ -n "$ROCM_LIBRARIES_COMMIT_OVERRIDE" ]; then \ - echo "Force updating rocm-libraries commit to ${ROCM_LIBRARIES_COMMIT_OVERRIDE}" && \ - git update-index --cacheinfo 160000,${ROCM_LIBRARIES_COMMIT_OVERRIDE},rocm-libraries ; \ - fi && \ - if [ -n "$ROCM_SYSTEMS_COMMIT_OVERRIDE" ]; then \ - echo "Force updating rocm-systems commit to ${ROCM_SYSTEMS_COMMIT_OVERRIDE}" && \ - git update-index --cacheinfo 160000,${ROCM_SYSTEMS_COMMIT_OVERRIDE},rocm-systems ; \ - fi - -FROM base AS build_rocm -ARG CCACHE_VERSION -ARG AMDGPU_FAMILIES -ARG AMDGPU_BUNDLE_NAME -COPY --from=setup_therock /app/TheRock /app/TheRock -WORKDIR /app/TheRock -# Pre-pin meson<1.12.0 before requirements.txt (floor: meson>=1.7.0), mirroring -# TheRock's manylinux Dockerfile. meson 1.12.0 duplicates env-derived link args -# (e.g. LDFLAGS=-Wl,--version-script=...) in its compiler sanity check, which -# `ld` rejects as a duplicate version tag and breaks the libdrm/libpciaccess -# sysdeps builds. See https://github.com/ROCm/TheRock/issues/7286. -RUN pip install --no-cache-dir "meson==1.7.0" -# if dvc is available, fetch_source.py uses it to fetch large files - install it along with other requirements. -# Mirroring TheRock's manylinux Dockerfile pin of aiobotocore so dvc[s3]'s -# botocore window stays aligned with the boto3 range pinned in -# requirements.txt (>=1.42.79,<1.42.85). aiobotocore==3.4.0 requires -# botocore>=1.42.79,<1.42.85; resolving everything in one pip call lets -# pip pick a botocore (1.42.84) that satisfies both pins. -RUN pip install --no-cache-dir -r requirements.txt "pathspec<0.13.0" "dvc[s3]==3.67.1" "aiobotocore==3.4.0" -RUN bash dockerfiles/install_ccache.sh ${CCACHE_VERSION} -RUN bash dockerfiles/install_pinned_patchelf.sh -RUN --mount=type=cache,target=/app/TheRock/.ccache,id=therock-ccache,sharing=locked \ - --mount=type=cache,target=/app/TheRock/git-mirror,id=therock-git-mirrors,sharing=locked \ - python build_tools/setup_git_mirrors.py --mirror-dir /app/TheRock/git-mirror --update --jobs 8 && \ - python build_tools/fetch_sources.py --jobs 8 --depth 1 --reference-dir /app/TheRock/git-mirror && \ - eval "$(python build_tools/setup_ccache.py --init --local)" && \ - ccache -o max_size=50G && \ - ccache -o sloppiness=include_file_ctime,include_file_mtime && \ - ccache -o compiler_check=content && \ - cmake -S /app/TheRock -B /app/TheRock/build -GNinja . \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_HIP_COMPILER_LAUNCHER=ccache \ - -DTHEROCK_AMDGPU_FAMILIES="${AMDGPU_FAMILIES}" \ - -DTHEROCK_TEST_AMDGPU_FAMILIES="${AMDGPU_FAMILIES}" \ - -DTHEROCK_BUNDLE_SYSDEPS=ON -DTHEROCK_AMDGPU_DIST_BUNDLE_NAME="${AMDGPU_BUNDLE_NAME}" \ - -DTHEROCK_ENABLE_ALL=OFF \ - -DTHEROCK_ENABLE_CORE=ON \ - -DTHEROCK_ENABLE_COMM_LIBS=ON \ - -DTHEROCK_ENABLE_DEBUG_TOOLS=ON \ - -DTHEROCK_ENABLE_MATH_LIBS=ON \ - -DTHEROCK_ENABLE_ML_LIBS=ON \ - -DTHEROCK_ENABLE_PROFILER=ON \ - -DTHEROCK_ENABLE_DC_TOOLS=ON \ - -DTHEROCK_ENABLE_MEDIA_LIBS=ON && \ - cmake --build /app/TheRock/build --target therock-artifacts && \ - echo "CCache stats:" && ccache -s -v && \ - mv /app/TheRock/build/artifacts /app/TheRock/artifacts -COPY .ci/collect_image_licenses.py /usr/local/libexec/collect_image_licenses.py -RUN python3 /usr/local/libexec/collect_image_licenses.py \ - --output /license-evidence \ - --scan-root therock=/app/TheRock \ - --git-root /app/TheRock \ - --fail-on-missing-git-license \ - --required-git-depth 0 +# rocm_tree and rocm_base install the same packages from `base`. The archive +# delete has to be inside the install it applies to: a child cannot drop +# files from a parent layer, and rocm_devel needs the archives this stage keeps. +FROM base AS rocm_tree +ARG ROCM_RELEASE_ID +ARG ROCM_DEB_SERIES +ARG ROCM_GFX_TARGETS +COPY docker/install_rocm_debs.sh /usr/local/bin/install_rocm_debs +COPY docker/setup_amdsmi.sh /usr/local/bin/setup_amdsmi +COPY docker/setup_rocm_llvm_layout.sh /usr/local/bin/setup_rocm_llvm_layout +COPY docker/setup_origami_cmake.sh /usr/local/bin/setup_origami_cmake +RUN chmod +x /usr/local/bin/install_rocm_debs \ + /usr/local/bin/setup_amdsmi \ + /usr/local/bin/setup_rocm_llvm_layout \ + /usr/local/bin/setup_origami_cmake \ + && install_rocm_debs base keep-archives +# The nightly deb ships the amd-smi Python bindings somewhere pip cannot reach; +# link them into site-packages. Prints a RETIRE banner and does nothing once a +# nightly makes `import amdsmi` work on its own. +RUN setup_amdsmi +ENV PATH="/opt/rocm/lib/llvm/bin:/opt/rocm/bin:$PATH" \ + ROCM_HOME=/opt/rocm \ + ROCM_PATH=/opt/rocm \ + HIP_CLANG_PATH=/opt/rocm/lib/llvm/bin \ + HIP_DEVICE_LIB_PATH=/opt/rocm/lib/llvm/amdgcn/bitcode \ + TRITON_HIP_LLD_PATH=/opt/rocm/lib/llvm/bin/ld.lld +# Nightly debs leave $ROCM_PATH/llvm/bin as a stub; FlyDSL/MLIR still +# invoke ld.lld there. Prints a RETIRE banner and does nothing once a +# nightly ships that path itself. Also retire if FlyDSL stops looking +# there. PATH is the ENV above, not this shim. +RUN setup_rocm_llvm_layout +# hipBLASLt's CMake config find_dependency(origami) next to hipblaslt; +# nightly debs ship liborigami.so without lib/cmake/origami. Prints a +# RETIRE banner and does nothing once a nightly ships that package. +RUN setup_origami_cmake -FROM base AS rocm_runtime -COPY docker/rocm_runtime_artifacts.txt /tmp/ -RUN --mount=type=bind,from=build_rocm,source=/app/TheRock,target=/app/TheRock/ \ - set -eu; \ - A="/app/TheRock/artifacts"; O="/opt/rocm"; \ - PREFIXES=$(sed '/^#/d; /^$/d' /tmp/rocm_runtime_artifacts.txt | tr '\n' ' '); \ - python3 /app/TheRock/build_tools/fileset_tool.py artifact-flatten-split \ - -o "${O}" --artifacts-dir "${A}" ${PREFIXES} -# Install amdsmi Python module into system site-packages. TheRock stages -# sources under share/amd_smi/amdsmi/ without setup.py (packaging rework in -# rocm-systems 5bc651a8+); pip install from share/amd_smi no longer works. -RUN AMDSMI_SITELIB="$(python3 -c "import site; print([p for p in site.getsitepackages() if not p.startswith('/usr/local')][0])")" && \ - cp -r /opt/rocm/share/amd_smi/amdsmi "${AMDSMI_SITELIB}/" -RUN echo "/opt/rocm/lib" > /etc/ld.so.conf.d/rocm.conf && \ - echo "/opt/rocm/lib/rocm_sysdeps/lib/" > /etc/ld.so.conf.d/rocm_sysdeps.conf && \ - ldconfig -ENV PATH="/opt/rocm/bin:$PATH" \ +FROM base AS rocm_base +ARG ROCM_RELEASE_ID +ARG ROCM_DEB_SERIES +ARG ROCM_GFX_TARGETS +COPY docker/install_rocm_debs.sh /usr/local/bin/install_rocm_debs +COPY docker/setup_amdsmi.sh /usr/local/bin/setup_amdsmi +COPY docker/setup_rocm_llvm_layout.sh /usr/local/bin/setup_rocm_llvm_layout +COPY docker/setup_origami_cmake.sh /usr/local/bin/setup_origami_cmake +RUN chmod +x /usr/local/bin/install_rocm_debs \ + /usr/local/bin/setup_amdsmi \ + /usr/local/bin/setup_rocm_llvm_layout \ + /usr/local/bin/setup_origami_cmake \ + && install_rocm_debs base strip-archives +RUN setup_amdsmi +ENV PATH="/opt/rocm/lib/llvm/bin:/opt/rocm/bin:$PATH" \ ROCM_HOME=/opt/rocm \ ROCM_PATH=/opt/rocm \ - HIP_DEVICE_LIB_PATH=/opt/rocm/lib/llvm/amdgcn/bitcode + HIP_CLANG_PATH=/opt/rocm/lib/llvm/bin \ + HIP_DEVICE_LIB_PATH=/opt/rocm/lib/llvm/amdgcn/bitcode \ + TRITON_HIP_LLD_PATH=/opt/rocm/lib/llvm/bin/ld.lld +RUN setup_rocm_llvm_layout +RUN setup_origami_cmake -FROM rocm_runtime AS rocm_devel -COPY docker/rocm_devel_artifacts.txt /tmp/ -RUN --mount=type=bind,from=build_rocm,source=/app/TheRock,target=/app/TheRock/ \ - set -eu; \ - A="/app/TheRock/artifacts"; O="/opt/rocm"; \ - PREFIXES=$(sed '/^#/d; /^$/d' /tmp/rocm_devel_artifacts.txt | tr '\n' ' '); \ - python3 /app/TheRock/build_tools/fileset_tool.py artifact-flatten-split \ - -o "${O}" --artifacts-dir "${A}" ${PREFIXES} -RUN ldconfig +FROM rocm_tree AS rocm_devel +ARG ROCM_RELEASE_ID +ARG ROCM_DEB_SERIES +ARG ROCM_GFX_TARGETS +RUN install_rocm_debs devel ENV PATH="/opt/rocm/libexec/rocprofiler-compute:$PATH" -FROM rocm_devel AS build_torch_stack +FROM rocm_tree AS build_torch_stack +ARG THEROCK_COMMIT ARG TRITON_COMMIT ARG PYTORCH_COMMIT ARG PYTORCH_VISION_COMMIT ARG PYTORCH_AUDIO_COMMIT ARG PYTORCH_CODEC_COMMIT ARG PYTORCH_ROCM_ARCH -COPY --from=setup_therock /app/TheRock /app/TheRock +ARG DEBIAN_FRONTEND=noninteractive + +# CPU BLAS headers for the PyTorch wheel. Runtime libs are installed in core. +RUN apt-get update -qq && apt-get install -y --no-install-recommends \ + libopenblas-dev \ + pkg-config && \ + rm -rf /var/lib/apt/lists/* + +# TheRock is only needed for external-builds/pytorch checkout scripts. +WORKDIR /app +RUN git init TheRock && \ + cd TheRock && \ + git remote add origin https://github.com/ROCm/TheRock.git && \ + git fetch --depth 1 origin ${THEROCK_COMMIT} && \ + git checkout --detach FETCH_HEAD # needed for HIPIFY commits created by the build process RUN git config --global user.name "TheRock Builder" && \ @@ -453,25 +398,22 @@ RUN python3 /usr/local/libexec/collect_image_licenses.py \ --output /ffmpeg/license-evidence \ --scan-root ffmpeg=${FFMPEG_SRC_DIR} -FROM rocm_runtime AS core_runtime +FROM ${ROCM_PARENT} AS core ARG AITER_COMMIT RUN --mount=type=bind,from=build_torch_stack,source=/install,target=/install \ pip install --no-cache-dir /install/*.whl -# AITER JIT: install dev artifacts required for runtime kernel compilation -RUN --mount=type=bind,from=build_rocm,source=/app/TheRock,target=/app/TheRock/ \ - set -eu; \ - A="/app/TheRock/artifacts"; O="/opt/rocm"; \ - python3 /app/TheRock/build_tools/fileset_tool.py artifact-flatten-split \ - -o "${O}" --artifacts-dir "${A}" \ - base_dev core-hip_dev core-runtime_dev blas_dev prim_dev rand_dev -# torchcodec: install FFmpeg and runtime dependencies +# Runtime libs for torchcodec/FFmpeg, OpenBLAS-linked torch, and OpenCV/GLib. RUN apt-get update -qq && apt-get -y install --no-install-recommends \ libaom3 \ libass9 \ libdav1d7 \ libfreetype6 \ + libgfortran5 \ + libgl1 \ + libglib2.0-0t64 \ libgnutls30 \ libmp3lame0 \ + libopenblas0 \ libopenh264-7 \ libvorbis0a \ libvorbisenc2 \ @@ -485,8 +427,7 @@ COPY --from=ffmpeg_builder /ffmpeg/build/share /usr/local/share RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/ffmpeg.conf && \ ldconfig COPY docker/libkineto.conf /config/libkineto.conf -ENV TRITON_HIP_LLD_PATH=/opt/rocm/lib/llvm/bin/ld.lld \ - KINETO_CONFIG="/config/libkineto.conf" +ENV KINETO_CONFIG="/config/libkineto.conf" # install aiter WORKDIR /app/external # Keep the Triton wheel we built from TRITON_COMMIT against our PyTorch; otherwise @@ -502,22 +443,8 @@ RUN --mount=type=bind,source=patches,target=/patches \ git submodule update --init --recursive && \ pip install --no-cache-dir -r requirements.txt && \ pip install -e . - -FROM core_runtime AS core -# Layer full devel artifacts on top of core_runtime (overlapping JIT-dev -# archives are re-extracted idempotently). Uses the same manifest as rocm_devel. -COPY docker/rocm_devel_artifacts.txt /tmp/ -RUN --mount=type=bind,from=build_rocm,source=/app/TheRock,target=/app/TheRock/ \ - set -eu; \ - A="/app/TheRock/artifacts"; O="/opt/rocm"; \ - PREFIXES=$(sed '/^#/d; /^$/d' /tmp/rocm_devel_artifacts.txt | tr '\n' ' '); \ - python3 /app/TheRock/build_tools/fileset_tool.py artifact-flatten-split \ - -o "${O}" --artifacts-dir "${A}" ${PREFIXES} -RUN ldconfig -ENV PATH="/opt/rocm/libexec/rocprofiler-compute:$PATH" # Held in core rather than final so that no stage downstream of core has an edge # back into a builder stage. -COPY --from=build_rocm /license-evidence /opt/license-evidence/rocm-build-sources COPY --from=build_torch_stack /install/license-evidence /opt/license-evidence/torch-build-sources COPY --from=ffmpeg_builder /ffmpeg/license-evidence /opt/license-evidence/ffmpeg-build-sources @@ -586,10 +513,12 @@ RUN --mount=type=bind,source=patches,target=/patches \ git apply /patches/xfuser_sd3_joint_attn_kv_cache.patch && \ pip freeze | grep numpy > constraints.txt && \ pip install -c constraints.txt -e . -# rocprofiler-compute runtime dependencies +# rocprofiler-compute ships with rocm_devel. ROCM_PARENT=rocm_base has no tree. # Due to https://github.com/ROCm/rocm-systems/pull/2821 drop numpy to avoid installing numpy<2 -RUN sed -i '/numpy/d' ${ROCM_PATH}/libexec/rocprofiler-compute/requirements.txt && \ - pip install --no-cache-dir -r ${ROCM_PATH}/libexec/rocprofiler-compute/requirements.txt +RUN if [ -f "${ROCM_PATH}/libexec/rocprofiler-compute/requirements.txt" ]; then \ + sed -i '/numpy/d' "${ROCM_PATH}/libexec/rocprofiler-compute/requirements.txt" && \ + pip install --no-cache-dir -r "${ROCM_PATH}/libexec/rocprofiler-compute/requirements.txt"; \ + fi # imageio + imageio-ffmpeg for diffusers export_to_video H.264 encoding # --no-binary avoids shipping imageio-ffmpeg's bundled GPL ffmpeg binary RUN pip install --no-cache-dir --no-binary :all: imageio-ffmpeg @@ -633,7 +562,6 @@ RUN python3 /usr/local/libexec/collect_image_licenses.py \ --scan-root python=/venv \ --scan-root rocm=/opt/rocm \ --scan-root local=/usr/local \ - --evidence-root rocm-build-sources=/opt/license-evidence/rocm-build-sources \ --evidence-root torch-build-sources=/opt/license-evidence/torch-build-sources \ --evidence-root ffmpeg-build-sources=/opt/license-evidence/ffmpeg-build-sources \ --git-root /app \ diff --git a/docker/README.md b/docker/README.md index 5eb47b37..098caf9d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,3 +6,106 @@ - `Dockerfile.sgld_lmsys` SGL-D image based on `lmsysorg/sglang-rocm` with CI scripts from xDiT - `Dockerfile.vllm_omni` vLLM-Omni image for ROCm based on `amdsiloai/pytorch-xdit` - `Dockerfile.vllm_omni.cuda` vLLM-Omni image for CUDA based on `vllm/vllm-openai` + +## How-tos + +### ROCm nightly packages + +`Dockerfile.ci` installs ROCm from a pinned TheRock nightly deb snapshot on the +[legacy multi-arch native index](https://rocm.nightlies.amd.com/packages-multi-arch/deb/). +Update `ROCM_RELEASE_ID` and `ROCM_DEB_SERIES` together when moving to a new +snapshot or ROCm release series. `ROCM_GFX_TARGETS` controls which architecture-specific +package shards are installed and is independent of `PYTORCH_ROCM_ARCH`, which +controls the architectures built into PyTorch and related wheels. + +Leaf packages install under `/opt/rocm/core-`. `rocm_tree` and `rocm_base` +both register the unversioned `/opt/rocm/{bin,lib,include,...}` links that the +`amdrocm-core` postinst would have created. They are separate installs from +`base`: `rocm_base` drops LLVM static archives outside `lib/clang` in that same +install step, and `rocm_tree` keeps them so `rocm_devel` can inherit the files. + +| Target | Packages | Use | +| --- | --- | --- | +| `rocm_tree` | HIP/`hipcc`, BLAS, MIOpen, RCCL, FFT, RAND, sparse, and solver for each arch in `ROCM_GFX_TARGETS`, plus headers. LLVM static archives kept | Parent of `build_torch_stack` and `rocm_devel` | +| `rocm_base` | Same packages. LLVM static archives outside `lib/clang` removed | Lean runtime target | +| `rocm_devel` | `rocm_tree` plus developer-tools, RDC, OpenCL, blas/rccl tests | Default parent of `core` / `deps` / `final` | + +`amdrocm-developer-tools` pulls the profiler, emulator, and debugger. It does +not depend on `amdrocm-core`. `core` is `FROM ${ROCM_PARENT}`, which defaults to +`rocm_devel`. Both parents install the wheels built by `build_torch_stack`. +`deps` installs rocprofiler-compute's Python requirements only when that tree +is present. + +```sh +# developer tools included +docker build -f docker/Dockerfile.ci --target final -t pytorch-xdit-dev . + +# runtime + torch stack + AITER JIT, without developer tools +docker build -f docker/Dockerfile.ci --target final \ + --build-arg ROCM_PARENT=rocm_base -t pytorch-xdit-lean . +``` + +The image does not build ROCm from source and does not support +`rocm-libraries` or `rocm-systems` commit overrides. Changes that are not +available in the pinned nightly must first be published in a nightly snapshot. + +### amd-smi Python bindings + +`docker/setup_amdsmi.sh` links the ROCm Python bindings into the venv, because +the nightly debs ship them under `${ROCM_HOME}/share/amd_smi/amdsmi` with no +`setup.py`, where pip cannot install them and nothing can import them. It runs +in `rocm_tree` and `rocm_base`; the script header explains why a symlink is the only correct +mechanism. + +If a future nightly puts the bindings on `sys.path` itself, a build log prints +a `setup_amdsmi: RETIRE THIS SHIM` banner and the script makes no changes — +delete it and its `COPY`/`RUN` at that point. Watch for the banner when bumping `ROCM_RELEASE_ID`: + +```sh +docker build -f docker/Dockerfile.ci --target rocm_base . --progress=plain 2>&1 \ + | grep -i 'setup_amdsmi' +``` + +### LLVM classic toolkit path + +`docker/setup_rocm_llvm_layout.sh` links `${ROCM_PATH}/lib/llvm/bin` tools into +`${ROCM_PATH}/llvm/bin`. Nightly debs install `ld.lld` under `lib/llvm/bin` and +leave `llvm/bin` as clang `.cfg` stubs; FlyDSL/MLIR still invoke +`$ROCM_PATH/llvm/bin/ld.lld`. It runs in `rocm_tree` and `rocm_base` after the ROCm `ENV` +block, which also puts `lib/llvm/bin` on `PATH`. + +If a future nightly ships an executable `ld.lld` at the classic path, a build +log prints a `setup_rocm_llvm_layout: RETIRE THIS SHIM` banner and the script +makes no changes — delete it and its `COPY`/`RUN` at that point. Keep the +`PATH=/opt/rocm/lib/llvm/bin` `ENV` unless the nightly also puts that directory +on `PATH`. The shim can also be deleted if FlyDSL (or its bundled MLIR) stops +looking for `$ROCM_PATH/llvm/bin/ld.lld` and uses `lib/llvm/bin`, +`HIP_CLANG_PATH`, `TRITON_HIP_LLD_PATH`, or `PATH` instead; that will not +print the banner. Watch for the banner when bumping `ROCM_RELEASE_ID`: + +```sh +docker build -f docker/Dockerfile.ci --target rocm_base . --progress=plain 2>&1 \ + | grep -i 'setup_rocm_llvm_layout' +``` + +### Origami CMake package + +`docker/setup_origami_cmake.sh` writes `${ROCM_HOME}/lib/cmake/origami/origami-config.cmake` +so `find_package(hipblaslt)` can configure. hipBLASLt's installed package config +requires `find_package(origami)` in a sibling `lib/cmake/origami` directory +(`NO_DEFAULT_PATH`, so `CMAKE_PREFIX_PATH` does not help). Nightly debs ship +`liborigami.so` in the BLAS host package but omit that CMake package from +BLAS devel. It runs in `rocm_tree` and `rocm_base` after the `-dev` debs, which is when +hipBLASLt's CMake files land. The script header explains why a stub imported +target is the only correct mechanism. + +If a future nightly ships `origami-config.cmake` (or `origamiConfig.cmake`) next +to hipBLASLt, a build log prints a `setup_origami_cmake: RETIRE THIS SHIM` +banner and the script makes no changes — delete it and its `COPY`/`RUN` at that +point. The shim can also be deleted if hipBLASLt stops `find_dependency(origami)`; +that will not print the banner. Watch for the banner when bumping `ROCM_RELEASE_ID`: + +```sh +docker build -f docker/Dockerfile.ci --target rocm_base . --progress=plain 2>&1 \ + | grep -i 'setup_origami_cmake' +``` diff --git a/docker/install_rocm_debs.sh b/docker/install_rocm_debs.sh new file mode 100755 index 00000000..5a69b41e --- /dev/null +++ b/docker/install_rocm_debs.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +set -euo pipefail + +readonly layer="${1:?usage: install_rocm_debs base strip-archives|keep-archives | devel}" +readonly archives="${2:-}" +readonly release_id="${ROCM_RELEASE_ID:?ROCM_RELEASE_ID must be set}" +readonly deb_series="${ROCM_DEB_SERIES:?ROCM_DEB_SERIES must be set}" +readonly gfx_targets="${ROCM_GFX_TARGETS:?ROCM_GFX_TARGETS must be set}" +readonly rocm_root="${ROCM_HOME:-/opt/rocm}" +readonly prefix="/opt/rocm/core-${deb_series}" +readonly repo_url="https://rocm.nightlies.amd.com/packages-multi-arch/deb/${release_id}" + +case "${layer}" in + base) + case "${archives}" in + strip-archives|keep-archives) ;; + *) + echo "base layer requires strip-archives or keep-archives" >&2 + exit 2 + ;; + esac + ;; + devel) + if [[ -n "${archives}" ]]; then + echo "devel layer takes no archive mode" >&2 + exit 2 + fi + ;; + *) + echo "unknown layer '${layer}'; expected base or devel" >&2 + exit 2 + ;; +esac + +echo "deb [trusted=yes] ${repo_url} stable main" \ + > /etc/apt/sources.list.d/rocm-nightly.list + +IFS=';' read -r -a targets <<< "${gfx_targets}" +if [[ "${#targets[@]}" -eq 0 ]]; then + echo "ROCM_GFX_TARGETS must contain at least one target" >&2 + exit 2 +fi + +packages=() +case "${layer}" in + base) + # Compiler, HIP runtime, and headers. runtime-dev depends on llvm-dev. + # Host libraries the AITER JIT link line names, plus the runtimes + # libtorch_hip loads. Per-arch shards hold device code. solver/sparse + # gfx packages are empty edges pulled in by the BLAS shard. + # rocSHMEM, hipFile, rocJPEG, rocDecode, and RPP stay out: this + # libtorch does not link them, and amdrocm-core would pull them in. + packages+=( + "amdrocm-runtime-dev${deb_series}" + "amdrocm-ccl-dev${deb_series}" + "amdrocm-blas-dev${deb_series}" + "amdrocm-blas-host${deb_series}" + "amdrocm-hipblas-common-dev${deb_series}" + "amdrocm-solver-dev${deb_series}" + "amdrocm-solver-host${deb_series}" + "amdrocm-dnn-dev${deb_series}" + "amdrocm-fft-dev${deb_series}" + "amdrocm-rand-dev${deb_series}" + "amdrocm-sparse-dev${deb_series}" + "amdrocm-rccl-dev${deb_series}" + "amdrocm-amdsmi${deb_series}" + ) + for target in "${targets[@]}"; do + [[ -n "${target}" ]] || continue + packages+=( + "amdrocm-blas${deb_series}-${target}" + "amdrocm-dnn${deb_series}-${target}" + "amdrocm-rccl${deb_series}-${target}" + "amdrocm-rand${deb_series}-${target}" + "amdrocm-fft${deb_series}-${target}" + ) + done + ;; + devel) + # developer-tools depends on the profiler, emulator, and debugger. + # It does not depend on amdrocm-core, so this layer does not pull + # the libraries rocm_base left out. + packages=( + "amdrocm-developer-tools${deb_series}" + "amdrocm-rdc${deb_series}" + "amdrocm-opencl${deb_series}" + ) + for target in "${targets[@]}"; do + [[ -n "${target}" ]] || continue + packages+=( + "amdrocm-blas-test${deb_series}-${target}" + "amdrocm-rccl-test${deb_series}-${target}" + ) + done + ;; +esac + +if [[ "${#packages[@]}" -eq 0 ]]; then + echo "ROCM_GFX_TARGETS did not contain a usable target" >&2 + exit 2 +fi + +apt-get update -qq +apt-get install -y --no-install-recommends "${packages[@]}" +rm -rf /var/lib/apt/lists/* + +# Leaf packages install under /opt/rocm/core-. The unversioned +# /opt/rocm/{bin,lib,include,...} links normally come from the amdrocm-core +# postinst, which this image does not install. +if [[ "${layer}" == "base" ]]; then + test -x "${prefix}/bin/hipcc" + test -e "${prefix}/lib/libamdhip64.so" + test -d "${prefix}/lib/llvm/amdgcn/bitcode" + major="${deb_series%%.*}" + score=100 + update-alternatives --install /opt/rocm/core core "${prefix}" "${score}" + update-alternatives --install "/opt/rocm/core-${major}" "core-${major}" "${prefix}" "${score}" + update-alternatives --install /opt/rocm/bin rocm-bin "${prefix}/bin" "${score}" + update-alternatives --install /opt/rocm/lib rocm-lib "${prefix}/lib" "${score}" + update-alternatives --install /opt/rocm/include rocm-include "${prefix}/include" "${score}" + update-alternatives --install /opt/rocm/llvm rocm-llvm "${prefix}/llvm" "${score}" + update-alternatives --install /opt/rocm/amdgcn rocm-amdgcn "${prefix}/amdgcn" "${score}" + update-alternatives --install /opt/rocm/libexec rocm-libexec "${prefix}/libexec" "${score}" + update-alternatives --install /opt/rocm/share rocm-share "${prefix}/share" "${score}" +fi + +echo "${rocm_root}/lib" > /etc/ld.so.conf.d/rocm.conf +if [[ -d "${rocm_root}/lib/rocm_sysdeps/lib" ]]; then + echo "${rocm_root}/lib/rocm_sysdeps/lib" \ + > /etc/ld.so.conf.d/rocm_sysdeps.conf +fi +ldconfig + +# hipcc links libLLVM.so / libclang-cpp.so. llvm-dev is a hard dependency of +# runtime-dev and is mostly static archives, including the Flang frontend. +# AITER JIT and the torch link do not static-link LLVM. The host link still +# needs libclang_rt.builtins.a under lib/clang/, so that tree stays. +# strip-archives deletes in this same step as the install. A later layer +# would leave the archives in the image. keep-archives is the rocm_tree +# install that rocm_devel inherits. The package stays installed either way. +if [[ "${archives}" == "strip-archives" ]]; then + find "${prefix}/lib/llvm/lib" -name '*.a' -not -path '*/lib/clang/*' -delete +fi + +case "${layer}" in + base) + test -d "${rocm_root}/lib/llvm/amdgcn/bitcode" + test -e "${rocm_root}/lib/llvm/bin/clang++" + test -e "${rocm_root}/lib/libamdhip64.so" + test -d "${rocm_root}/lib/rocm_sysdeps/lib" + test -e "${rocm_root}/lib/librocprofiler-sdk.so" + test -e "${rocm_root}/lib/libroctracer64.so" + test -d "${rocm_root}/include/roctracer" + test -d "${rocm_root}/lib/rocm_sysdeps/include" + test -d "${rocm_root}/lib/rocm_sysdeps/lib/pkgconfig" + test -e "${rocm_root}/lib/libMIOpen.so" + test -e "${rocm_root}/lib/librccl.so" + test -e "${rocm_root}/lib/libhipblaslt.so" + frontend="${prefix}/lib/llvm/lib/libflangFrontend.a" + if [[ "${archives}" == "keep-archives" ]]; then + test -e "${frontend}" + else + test ! -e "${frontend}" + fi + ;; + devel) + test -d "${rocm_root}/libexec/rocprofiler-compute" + test -e "${prefix}/lib/llvm/lib/libflangFrontend.a" + ;; +esac diff --git a/docker/rocm_devel_artifacts.txt b/docker/rocm_devel_artifacts.txt deleted file mode 100644 index f0b4b30d..00000000 --- a/docker/rocm_devel_artifacts.txt +++ /dev/null @@ -1,102 +0,0 @@ -# ROCm devel/debug/test artifacts layered on top of the runtime set. -# Shared by the rocm_devel stage (for build_torch_stack) and the core stage. -# Lines starting with '#' are comments; blank lines are ignored. -# Each entry is an artifact prefix (_) consumed by -# fileset_tool.py artifact-flatten-split, which globs both the -# _generic shard and any per-arch shards produced by KPACK splitting. - -amd-dbgapi_dbg -amd-dbgapi_dev -amd-dbgapi_lib -amd-llvm_dbg -amd-llvm_dev -aqlprofile_dbg -aqlprofile_dev -base_dbg -base_dev -blas_dbg -blas_dev -# blas_test artifact is included because of hipblaslt-bench -blas_test -composable-kernel_dbg -composable-kernel_dev -core-amdsmi_dbg -core-amdsmi_dev -core-hip_dbg -core-hip_dev -core-kpack_dbg -core-kpack_dev -core-ocl_dbg -core-ocl_dev -core-ocl_lib -core-ocl_run -core-runtime_dbg -core-runtime_dev -fft_dbg -fft_dev -fftw3_dbg -fftw3_dev -hipblasltprovider_dbg -hipdnn_dbg -hipdnn_dev -hipify_dbg -hipify_run -host-blas_dbg -host-blas_dev -host-suite-sparse_dbg -host-suite-sparse_dev -libhipcxx_dbg -libhipcxx_dev -miopen_dbg -miopen_dev -miopenprovider_dbg -prim_dbg -prim_dev -rand_dbg -rand_dev -rccl_dbg -rccl_dev -# rccl_test artifact is included because of all_gather_perf and similar tools -rccl_test -rdc_dbg -rdc_dev -rdc_lib -rdc_run -rocdecode_dbg -rocdecode_dev -rocdecode_lib -rocgdb_dbg -rocgdb_dev -rocgdb_lib -rocgdb_run -rocjpeg_dbg -rocjpeg_dev -rocjpeg_lib -rocprofiler-compute_dbg -rocprofiler-compute_dev -rocprofiler-compute_lib -rocprofiler-compute_run -rocprofiler-sdk_dbg -rocprofiler-sdk_dev -rocprofiler-systems_dbg -rocprofiler-systems_dev -rocprofiler-systems_lib -rocprofiler-systems_run -rocr-debug-agent_dbg -rocr-debug-agent_dev -rocr-debug-agent_lib -rocrtst_dbg -rocrtst_dev -rocrtst_lib -rocrtst_run -rocwmma_dbg -rocwmma_dev -sysdeps_dev -sysdeps-amd-mesa_dev -sysdeps-amd-mesa_lib -sysdeps-expat_dev -sysdeps-gmp_dev -sysdeps-hwloc_dev -sysdeps-libpciaccess_dev -sysdeps-mpfr_dev -sysdeps-ncurses_dev diff --git a/docker/rocm_runtime_artifacts.txt b/docker/rocm_runtime_artifacts.txt deleted file mode 100644 index 94852ab4..00000000 --- a/docker/rocm_runtime_artifacts.txt +++ /dev/null @@ -1,56 +0,0 @@ -# ROCm runtime artifacts installed in the rocm_runtime stage. -# Lines starting with '#' are comments; blank lines are ignored. -# Each entry is an artifact prefix (_) consumed by -# fileset_tool.py artifact-flatten-split, which globs both the -# _generic shard and any per-arch shards produced by KPACK splitting. - -amd-llvm_lib -amd-llvm_run -# aqlprofile provides libhsa-amd-aqlprofile64.so, required at runtime by -# rocprofiler-sdk for PMC sampling and by anything loading rocprof -aqlprofile_lib -aqlprofile_run -base_lib -base_run -blas_lib -blas_run -composable-kernel_lib -core-amdsmi_lib -core-amdsmi_run -core-hip_lib -core-hip_run -core-kpack_lib -core-runtime_lib -core-runtime_run -fft_lib -fft_run -fftw3_lib -fftw3_run -hipblasltprovider_lib -hipdnn_lib -hipdnn_run -host-blas_lib -host-blas_run -host-suite-sparse_lib -host-suite-sparse_run -libhipcxx_lib -miopen_lib -miopen_run -miopenprovider_lib -rand_lib -rand_run -rccl_lib -rccl_run -# rocprofiler-sdk is needed at runtime: PyTorch Kineto (USE_KINETO=ON) depends -# on libroctracer64.so which in turn requires librocprofiler-sdk.so -rocprofiler-sdk_lib -rocprofiler-sdk_run -rocwmma_lib -sysdeps_lib -sysdeps_run -sysdeps-expat_lib -sysdeps-gmp_lib -sysdeps-hwloc_lib -sysdeps-libpciaccess_lib -sysdeps-mpfr_lib -sysdeps-ncurses_lib diff --git a/docker/setup_amdsmi.sh b/docker/setup_amdsmi.sh new file mode 100755 index 00000000..6a9c08b5 --- /dev/null +++ b/docker/setup_amdsmi.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Make `import amdsmi` work inside the venv. +# +# The amd-smi CLI lives at ${ROCM_HOME}/bin and is unaffected by this script. +# The nightly debs drop the Python bindings at ${ROCM_HOME}/share/amd_smi/amdsmi +# with no setup.py or pyproject.toml, so pip refuses to install them and they +# land nowhere on sys.path. torch.cuda wants them for device_count() and for +# utilization/memory_usage/temperature/power_draw/clock_rate; accelerate wants +# them for set_numa_affinity under ACCELERATE_CPU_AFFINITY. +# +# The fix is a symlink, never a copy. amdsmi_wrapper._load_library() finds the +# shared object through Path(__file__).resolve(), so a link resolves back into +# the ROCm tree and loads the matching lib/libamd_smi.so.; a copy +# breaks that and falls through to a bare-SONAME linker lookup. Putting +# share/amd_smi on sys.path instead (PYTHONPATH or a .pth file) is also wrong, +# because it exposes the sibling tools/, tests/ and example/ directories as +# implicit namespace packages. +# +# This shim is expected to expire. Once a nightly ships the bindings on sys.path +# on its own, the script changes nothing and prints a RETIRE banner instead -- +# the signal to delete it along with its COPY/RUN in docker/Dockerfile.ci. + +set -euo pipefail + +readonly rocm_root="${ROCM_HOME:-/opt/rocm}" +readonly src="${rocm_root}/share/amd_smi/amdsmi" + +site_packages="$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" +readonly site_packages +readonly link="${site_packages}/amdsmi" + +# Where does `import amdsmi` resolve from right now, if anywhere? +probe_origin() { + python3 -c \ + 'import importlib.util as u; print(getattr(u.find_spec("amdsmi"), "origin", "") or "")' \ + 2>/dev/null || true +} + +# Drop a link from an earlier run so the probe sees only what packaging +# provides. Anything else at that path came from elsewhere and is left alone. +if [[ -L "${link}" ]]; then + rm -f "${link}" +fi + +origin="$(probe_origin)" +if [[ -n "${origin}" ]]; then + echo "==============================================================" >&2 + echo " setup_amdsmi: RETIRE THIS SHIM" >&2 + echo " \`import amdsmi\` already resolves without it, from" >&2 + echo " ${origin}" >&2 + if [[ "${origin}" != "${rocm_root}/"* ]]; then + echo " Note: that path is outside ${rocm_root}, so the bindings are" >&2 + echo " probably a PyPI amdsmi wheel rather than a fixed nightly deb." >&2 + echo " Those bundle their own libamd_smi and can skew from ROCm." >&2 + fi + echo " No symlink was created. Delete docker/setup_amdsmi.sh and its" >&2 + echo " COPY/RUN in docker/Dockerfile.ci." >&2 + echo "==============================================================" >&2 + exit 0 +fi + +if [[ ! -d "${src}" ]]; then + echo "amdsmi is not importable and ${src} does not exist;" >&2 + echo "is the amdrocm-amdsmi deb installed in this layer?" >&2 + exit 1 +fi + +ln -sfT "${src}" "${link}" + +# Precompile in the ROCm tree so containers do not each rebuild __pycache__ on +# first import. Purely a nicety, so a failure here must not fail the build. +python3 -m compileall -q "${src}" > /dev/null || + echo "warning: could not precompile ${src}; imports compile at runtime" >&2 + +# The wrapper is import-tolerant: it swaps in a _MissingLibrary sentinel when +# the .so cannot be loaded, so a bare `import amdsmi` proves nothing. Check the +# library the wrapper actually resolved. +python3 - <<'PY' +import amdsmi +from amdsmi import amdsmi_wrapper + +missing = object() +lib = getattr(amdsmi_wrapper, "_loaded_lib_path", missing) +if lib is None: + raise SystemExit(f"linked {amdsmi.__file__} but libamd_smi.so failed to load") +if lib is missing: + print(f"linked {amdsmi.__file__} (library unverified: the wrapper no longer " + "exposes _loaded_lib_path)") +else: + print(f"linked {amdsmi.__file__} -> {lib}") +PY diff --git a/docker/setup_origami_cmake.sh b/docker/setup_origami_cmake.sh new file mode 100755 index 00000000..99de7a86 --- /dev/null +++ b/docker/setup_origami_cmake.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Provide a CMake package for origami so find_package(hipblaslt) can configure. +# +# hipBLASLt's installed hipblaslt-package-config.cmake does: +# find_dependency(origami CONFIG +# PATHS "${CMAKE_CURRENT_LIST_DIR}/../origami" +# "${CMAKE_CURRENT_LIST_DIR}/origami" +# NO_DEFAULT_PATH) +# CMAKE_PREFIX_PATH is ignored. Nightly debs currently ship liborigami.so in +# amdrocm-blas-host but omit lib/cmake/origami from amdrocm-blas-dev, so +# PyTorch's LoadHIP.cmake dies at find_package(hipblaslt REQUIRED). +# +# The fix is a one-file CMake config that imports the .so already on disk as +# roc::origami. It is not a copy of the upstream origami-config.cmake.in +# (that includes origami-targets.cmake and hip). +# +# This shim is expected to expire. Once a nightly ships origami-config.cmake +# (or origamiConfig.cmake) next to hipBLASLt on its own, the script changes +# nothing and prints a RETIRE banner instead -- the signal to delete it along +# with its COPY/RUN in docker/Dockerfile.ci. Independently, if hipBLASLt +# stops find_dependency(origami), this shim can be deleted even while the +# .so remains unpackaged as a CMake package; that case will not print the +# banner. + +set -euo pipefail + +readonly rocm_root="${ROCM_HOME:-${ROCM_PATH:-/opt/rocm}}" +readonly shim_marker="setup_origami_cmake: SHIM" +readonly dest_dir="${rocm_root}/lib/cmake/origami" +readonly dest="${dest_dir}/origami-config.cmake" + +# Paths hipBLASLt searches (NO_DEFAULT_PATH). First existing config wins. +config_candidates=( + "${rocm_root}/lib/cmake/origami/origami-config.cmake" + "${rocm_root}/lib/cmake/origami/origamiConfig.cmake" + "${rocm_root}/lib/cmake/hipblaslt/origami/origami-config.cmake" + "${rocm_root}/lib/cmake/hipblaslt/origami/origamiConfig.cmake" +) + +existing="" +for candidate in "${config_candidates[@]}"; do + if [[ -f "${candidate}" ]]; then + existing="${candidate}" + break + fi +done + +if [[ -n "${existing}" ]] && ! grep -q "${shim_marker}" "${existing}"; then + echo "==============================================================" >&2 + echo " setup_origami_cmake: RETIRE THIS SHIM" >&2 + echo " find_package(origami) already has a CMake config at" >&2 + echo " ${existing}" >&2 + echo " No stub was written. Delete docker/setup_origami_cmake.sh" >&2 + echo " and its COPY/RUN in docker/Dockerfile.ci." >&2 + echo "==============================================================" >&2 + exit 0 +fi + +lib="" +for candidate in \ + "${rocm_root}/lib/liborigami.so" \ + "${rocm_root}/lib/liborigami.so.1"; do + if [[ -e "${candidate}" ]]; then + lib="${candidate}" + break + fi +done + +if [[ -z "${lib}" ]]; then + echo "origami CMake config is missing and liborigami.so was not found" >&2 + echo "under ${rocm_root}/lib; is the amdrocm-blas-host deb installed" >&2 + echo "in this layer?" >&2 + exit 1 +fi + +mkdir -p "${dest_dir}" +cat > "${dest}" < $(readlink -f "${lib}")" diff --git a/docker/setup_rocm_llvm_layout.sh b/docker/setup_rocm_llvm_layout.sh new file mode 100755 index 00000000..ecfc9f4d --- /dev/null +++ b/docker/setup_rocm_llvm_layout.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Fill $ROCM_PATH/llvm/bin so FlyDSL/MLIR can find ld.lld. +# +# Nightly debs install the LLVM toolchain at ${ROCM_PATH}/lib/llvm/bin and leave +# ${ROCM_PATH}/llvm/bin as a stub of clang .cfg files. FlyDSL's bundled MLIR +# still serializes GPU binaries by invoking $ROCM_PATH/llvm/bin/ld.lld (classic +# toolkit layout), which fails with "lld invocation failed" when that path is +# missing. PATH lookups are a second, weaker contract; docker/Dockerfile.ci +# puts lib/llvm/bin on PATH via ENV and this script does not try to persist that. +# +# The fix is a symlink per missing name, never a copy, into the stub directory. +# Existing cfg files (and anything else already present) are left alone. +# +# This shim is expected to expire in either of two ways. Once a nightly ships +# an executable ld.lld under ${ROCM_PATH}/llvm/bin on its own, the script +# changes nothing and prints a RETIRE banner instead -- the signal to delete +# it along with its COPY/RUN in docker/Dockerfile.ci. Independently, if FlyDSL +# (or its bundled MLIR) stops looking at $ROCM_PATH/llvm/bin/ld.lld and uses +# lib/llvm/bin, HIP_CLANG_PATH, TRITON_HIP_LLD_PATH, or PATH instead, this +# shim can be deleted even while the stub directory remains; that case will +# not print the banner. PATH can stay: it is not this shim. + +set -euo pipefail + +readonly rocm_root="${ROCM_PATH:-${ROCM_HOME:-/opt/rocm}}" +readonly llvm_bin="${rocm_root}/lib/llvm/bin" +readonly classic_bin="${rocm_root}/llvm/bin" + +if [[ -x "${classic_bin}/ld.lld" ]]; then + echo "==============================================================" >&2 + echo " setup_rocm_llvm_layout: RETIRE THIS SHIM" >&2 + echo " ${classic_bin}/ld.lld is already executable:" >&2 + echo " $(readlink -f "${classic_bin}/ld.lld")" >&2 + echo " No extra tool symlinks were created. Delete" >&2 + echo " docker/setup_rocm_llvm_layout.sh and its COPY/RUN in" >&2 + echo " docker/Dockerfile.ci. Keep PATH=/opt/rocm/lib/llvm/bin in" >&2 + echo " the Dockerfile ENV unless a nightly also puts that on PATH." >&2 + echo "==============================================================" >&2 + exit 0 +fi + +if [[ ! -x "${llvm_bin}/ld.lld" ]]; then + echo "ld.lld is missing from both ${classic_bin} and ${llvm_bin};" >&2 + echo "is the amdrocm-core LLVM toolchain installed in this layer?" >&2 + exit 1 +fi + +mkdir -p "${classic_bin}" + +linked=0 +for src in "${llvm_bin}"/*; do + dest="${classic_bin}/$(basename "${src}")" + if [[ -e "${src}" && ! -e "${dest}" ]]; then + ln -s "${src}" "${dest}" + linked=$((linked + 1)) + fi +done + +if [[ ! -x "${classic_bin}/ld.lld" ]]; then + echo "linked ${linked} names from ${llvm_bin} into ${classic_bin}" >&2 + echo "but ${classic_bin}/ld.lld is still not executable" >&2 + exit 1 +fi + +echo "linked ${linked} LLVM toolchain names ${llvm_bin} -> ${classic_bin}" +echo "ld.lld -> $(readlink -f "${classic_bin}/ld.lld")"