diff --git a/.cargo/config.toml b/.cargo/config.toml index b00534a..e9143fc 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,5 +1,7 @@ [alias] -xtask = "run --manifest-path ./xtask/Cargo.toml --" +xtask = "run --locked --manifest-path ./xtask/Cargo.toml --" -[target.x86_64-unknown-linux-musl] -rustflags = ["-Z", "remap-cwd-prefix=", "-C", "panic=abort"] +rustflags = ["-Z", "remap-cwd-prefix="] + +[unstable] +build-std = ["std", "panic_abort"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa5dd8f..9947860 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,42 +3,354 @@ name: CI on: push: branches: - - action + - action tags: - - '*' + - '*' workflow_dispatch: +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - build_and_release: - name: uruntime - runs-on: ubuntu-latest - permissions: - contents: write + preflight: + name: Validate pinned downloads + runs-on: ubuntu-24.04 steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install preflight dependencies + run: | + sudo apt-get \ + -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources \ + -o Dir::Etc::sourceparts=- \ + update + sudo apt-get \ + -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources \ + -o Dir::Etc::sourceparts=- \ + install --yes llvm musl-tools python3-yaml + rustup component add rust-src rustfmt clippy + rustup target add x86_64-unknown-linux-musl + + - name: Run local quality gates and checksum validation + run: cargo --locked xtask check + + - name: Test CI artifact and workflow contracts + env: + PYTHONDONTWRITEBYTECODE: '1' + run: python3 -m unittest discover -s tests -p 'test_ci_*.py' -v + build: + name: Build ${{ matrix.arch }} + needs: preflight + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + arch: + - x86_64 + - aarch64 + - riscv64 + - loongarch64 + - ppc64 + - ppc64le + steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install build dependencies + run: | + sudo apt-get \ + -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources \ + -o Dir::Etc::sourceparts=- \ + update + sudo apt-get \ + -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources \ + -o Dir::Etc::sourceparts=- \ + install --yes binutils curl llvm musl-tools xz-utils + rustup component add rust-src + + - name: Build all runtime variants + run: cargo --locked xtask ${{ matrix.arch }} + + - name: Install QEMU for foreign smoke tests + if: matrix.arch != 'x86_64' + run: sudo apt-get install --yes qemu-user-static + + - name: Validate artifact manifest, ELF metadata, and smoke tests + run: python3 scripts/ci_artifacts.py validate-arch '${{ matrix.arch }}' dist --smoke - - name: Setup toolchain - uses: actions-rs/toolchain@v1 + - name: Upload architecture artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - toolchain: nightly - override: true - target: x86_64-unknown-linux-musl - - - name: Install deps - run: > - sudo bash -c 'apt update && apt install binutils curl musl-tools qemu-user-static llvm -y'; - rustup component add rust-src --toolchain nightly; - cargo install cross; - - - name: Build - run: cargo xtask all - - - name: Release - uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/') + name: uruntime-${{ matrix.arch }} + path: dist/uruntime-*-${{ matrix.arch }} + if-no-files-found: error + compression-level: 0 + retention-days: 1 + + release: + name: Release + needs: build + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Download all architecture artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - files: dist/uruntime* + pattern: uruntime-* + path: artifacts + + - name: Validate and stage the exact release manifest + run: python3 scripts/ci_artifacts.py aggregate-release artifacts release-dist + + - name: Verify the remote tag identifies this workflow commit + env: + RELEASE_TAG: ${{ github.ref_name }} + EXPECTED_SHA: ${{ github.sha }} + run: | + set -euo pipefail + direct='' + peeled='' + while read -r sha ref; do + if [[ "$ref" == "refs/tags/$RELEASE_TAG" ]]; then + direct=$sha + elif [[ "$ref" == "refs/tags/$RELEASE_TAG^{}" ]]; then + peeled=$sha + fi + done < <(git ls-remote origin "refs/tags/$RELEASE_TAG" "refs/tags/$RELEASE_TAG^{}") + remote_sha=${peeled:-$direct} + if [[ -z "$remote_sha" || "$remote_sha" != "$EXPECTED_SHA" ]]; then + printf 'remote tag %s resolves to %s, expected %s\n' \ + "$RELEASE_TAG" "${remote_sha:-}" "$EXPECTED_SHA" >&2 + exit 1 + fi + + - name: Create or refresh the exact-tag draft release + id: acquire_release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.ref_name }} + EXPECTED_SHA: ${{ github.sha }} + RELEASE_MARKER: '' + run: | + set -euo pipefail + releases_file=$(mktemp) + response_file=$(mktemp) + trap 'rm -f "$releases_file" "$response_file"' EXIT + gh api --paginate --slurp \ + "repos/$GITHUB_REPOSITORY/releases?per_page=100" > "$releases_file" + release_id=$(python3 - "$releases_file" "$RELEASE_TAG" <<'PY' + import json + import sys + + with open(sys.argv[1], encoding="utf-8") as source: + pages = json.load(source) + if not isinstance(pages, list) or not all(isinstance(page, list) for page in pages): + raise SystemExit("invalid paginated release-list response") + matches = [release for page in pages for release in page + if isinstance(release, dict) and release.get("tag_name") == sys.argv[2]] + if len(matches) > 1: + raise SystemExit(f"multiple releases found for exact tag {sys.argv[2]!r}") + if matches: + release_id = matches[0].get("id") + if isinstance(release_id, bool) or not isinstance(release_id, int) or release_id <= 0: + raise SystemExit("exact-tag release has an invalid numeric ID") + print(release_id) + PY + ) + release_body=$(printf '%s\n\n%s' "$(cat RELEASE_NOTES.md)" "$RELEASE_MARKER") + if [[ -n "$release_id" ]]; then + [[ $release_id =~ ^[0-9]+$ ]] + gh api --method PATCH "repos/$GITHUB_REPOSITORY/releases/$release_id" \ + -f "tag_name=$RELEASE_TAG" \ + -f "target_commitish=$EXPECTED_SHA" \ + -f "name=$RELEASE_TAG" \ + -f "body=$release_body" \ + -F draft=true \ + -F prerelease=false \ + -F generate_release_notes=false \ + -f make_latest=false > "$response_file" + release_action=refreshed + else + gh api --method POST "repos/$GITHUB_REPOSITORY/releases" \ + -f "tag_name=$RELEASE_TAG" \ + -f "target_commitish=$EXPECTED_SHA" \ + -f "name=$RELEASE_TAG" \ + -f "body=$release_body" \ + -F draft=true \ + -F prerelease=false \ + -F generate_release_notes=false \ + -f make_latest=false > "$response_file" + release_id=$(jq -er '.id | select(type == "number" and . > 0)' "$response_file") + [[ $release_id =~ ^[0-9]+$ ]] + release_action=created + fi + jq -e \ + --argjson id "$release_id" --arg tag "$RELEASE_TAG" \ + --arg sha "$EXPECTED_SHA" --arg marker "$RELEASE_MARKER" \ + '.id == $id and .tag_name == $tag and .target_commitish == $sha and + .draft == true and .prerelease == false and + (.body | type == "string" and contains($marker))' \ + "$response_file" > /dev/null + printf 'release_id=%s\n' "$release_id" >> "$GITHUB_OUTPUT" + printf 'release_action=%s\n' "$release_action" >> "$GITHUB_OUTPUT" + + - name: Delete every old asset from the exact draft release ID + env: + GH_TOKEN: ${{ github.token }} + RELEASE_ID: ${{ steps.acquire_release.outputs.release_id }} + RELEASE_TAG: ${{ github.ref_name }} + RELEASE_MARKER: '' + run: | + set -euo pipefail + [[ $RELEASE_ID =~ ^[0-9]+$ ]] + gh api "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID" > release-before-delete.json + jq -e \ + --argjson id "$RELEASE_ID" --arg tag "$RELEASE_TAG" --arg marker "$RELEASE_MARKER" \ + '.id == $id and .tag_name == $tag and .draft == true and + (.body | type == "string" and contains($marker))' \ + release-before-delete.json > /dev/null + gh api --paginate --slurp \ + "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID/assets?per_page=100" \ + > old-release-assets-pages.json + python3 - old-release-assets-pages.json old-release-asset-ids.txt <<'PY' + import json + import sys + + with open(sys.argv[1], encoding="utf-8") as source: + pages = json.load(source) + if not isinstance(pages, list) or not all(isinstance(page, list) for page in pages): + raise SystemExit("invalid paginated asset-list response") + assets = [asset for page in pages for asset in page] + ids = [] + for asset in assets: + asset_id = asset.get("id") if isinstance(asset, dict) else None + if isinstance(asset_id, bool) or not isinstance(asset_id, int) or asset_id <= 0: + raise SystemExit("release asset has an invalid numeric ID") + ids.append(asset_id) + with open(sys.argv[2], "w", encoding="ascii") as output: + for asset_id in ids: + print(asset_id, file=output) + PY + while IFS= read -r asset_id; do + [[ $asset_id =~ ^[0-9]+$ ]] + gh api --method DELETE "repos/$GITHUB_REPOSITORY/releases/assets/$asset_id" + done < old-release-asset-ids.txt + gh api --paginate --slurp \ + "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID/assets?per_page=100" \ + | jq -e 'all(.[]; length == 0)' > /dev/null + + - name: Upload exactly 54 assets to the draft release ID + env: + GH_TOKEN: ${{ github.token }} + RELEASE_ID: ${{ steps.acquire_release.outputs.release_id }} + run: | + set -euo pipefail + [[ $RELEASE_ID =~ ^[0-9]+$ ]] + shopt -s nullglob + files=(release-dist/uruntime-*) + [[ ${#files[@]} -eq 54 ]] + for file in "${files[@]}"; do + [[ -f "$file" && ! -L "$file" ]] + name=${file##*/} + encoded_name=$(jq -rn --arg value "$name" '$value | @uri') + curl --fail-with-body --show-error --silent --location \ + --request POST \ + --header "Authorization: Bearer $GH_TOKEN" \ + --header 'Accept: application/vnd.github+json' \ + --header 'X-GitHub-Api-Version: 2022-11-28' \ + --header 'Content-Type: application/octet-stream' \ + --data-binary "@$file" \ + "https://uploads.github.com/repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID/assets?name=$encoded_name" \ + > /dev/null + done + + - name: Verify the exact paginated asset manifest by release ID + env: + GH_TOKEN: ${{ github.token }} + RELEASE_ID: ${{ steps.acquire_release.outputs.release_id }} + RELEASE_TAG: ${{ github.ref_name }} + RELEASE_MARKER: '' + run: | + set -euo pipefail + [[ $RELEASE_ID =~ ^[0-9]+$ ]] + gh api "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID" > release-before-publish.json + jq -e \ + --argjson id "$RELEASE_ID" --arg tag "$RELEASE_TAG" --arg marker "$RELEASE_MARKER" \ + '.id == $id and .tag_name == $tag and .draft == true and + (.body | type == "string" and contains($marker))' \ + release-before-publish.json > /dev/null + gh api --paginate --slurp \ + "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID/assets?per_page=100" \ + > release-assets-pages.json + python3 -c 'import json,sys; json.dump([a for p in json.load(open(sys.argv[1])) for a in p], open(sys.argv[2], "w"))' \ + release-assets-pages.json release-assets.json + python3 scripts/ci_artifacts.py validate-release release-assets.json + + - name: Re-verify the remote tag immediately before publishing + env: + RELEASE_TAG: ${{ github.ref_name }} + EXPECTED_SHA: ${{ github.sha }} + run: | + set -euo pipefail + direct='' + peeled='' + while read -r sha ref; do + if [[ "$ref" == "refs/tags/$RELEASE_TAG" ]]; then + direct=$sha + elif [[ "$ref" == "refs/tags/$RELEASE_TAG^{}" ]]; then + peeled=$sha + fi + done < <(git ls-remote origin "refs/tags/$RELEASE_TAG" "refs/tags/$RELEASE_TAG^{}") + remote_sha=${peeled:-$direct} + if [[ -z "$remote_sha" || "$remote_sha" != "$EXPECTED_SHA" ]]; then + printf 'remote tag %s resolves to %s, expected %s\n' \ + "$RELEASE_TAG" "${remote_sha:-}" "$EXPECTED_SHA" >&2 + exit 1 + fi + + - name: Publish only the verified exact-tag release ID env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ github.token }} + RELEASE_ID: ${{ steps.acquire_release.outputs.release_id }} + RELEASE_TAG: ${{ github.ref_name }} + EXPECTED_SHA: ${{ github.sha }} + RELEASE_MARKER: '' + run: | + set -euo pipefail + [[ $RELEASE_ID =~ ^[0-9]+$ ]] + release_body=$(printf '%s\n\n%s' "$(cat RELEASE_NOTES.md)" "$RELEASE_MARKER") + gh api --method PATCH "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID" \ + -f "tag_name=$RELEASE_TAG" \ + -f "target_commitish=$EXPECTED_SHA" \ + -f "name=$RELEASE_TAG" \ + -f "body=$release_body" \ + -F draft=false \ + -F prerelease=false \ + -F generate_release_notes=false \ + -f make_latest=legacy > published-release.json + jq -e \ + --argjson id "$RELEASE_ID" --arg tag "$RELEASE_TAG" \ + --arg sha "$EXPECTED_SHA" --arg marker "$RELEASE_MARKER" \ + '.id == $id and .tag_name == $tag and .target_commitish == $sha and + .draft == false and .prerelease == false and + (.body | type == "string" and contains($marker))' \ + published-release.json > /dev/null + gh api "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID" > published-release-readback.json + jq -e \ + --argjson id "$RELEASE_ID" --arg tag "$RELEASE_TAG" \ + --arg sha "$EXPECTED_SHA" --arg marker "$RELEASE_MARKER" \ + '.id == $id and .tag_name == $tag and .target_commitish == $sha and + .draft == false and .prerelease == false and + (.body | type == "string" and contains($marker))' \ + published-release-readback.json > /dev/null diff --git a/.gitignore b/.gitignore index 4ae832f..4fc960c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,19 @@ -dist -assets* -target -.vscode -Cargo.lock +/dist/ +/release-dist/ +/artifacts/ + +/assets +/assets-*/ + +/target/ +/xtask/target/ + +/.vscode/ +/.ruff_cache/ +/.pytest_cache/ +__pycache__/ +*.py[cod] + +/release.json +/release-assets.json +/release-assets-pages.json diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..ee61546 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,744 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "cc" +version = "1.2.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chrono" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dotenv" +version = "0.16.0" +source = "git+https://github.com/VHSgunzo/dotenv.git?rev=367085af80a8d29d2241c85cbfc25736486230c5#367085af80a8d29d2241c85cbfc25736486230c5" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" + +[[package]] +name = "flate2" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "goblin" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17582616a7718cca54cec18e534a76c7c4aec11a8b9a85695712f262fd15a4c8" +dependencies = [ + "log", + "plain", + "scroll", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "iana-time-zone" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "memfd-exec" +version = "0.2.6" +source = "git+https://github.com/VHSgunzo/memfd-exec.git?rev=2decf7d1cec3d183e55000526abd2e5bb6df40c5#2decf7d1cec3d183e55000526abd2e5bb6df40c5" +dependencies = [ + "libc", + "nix", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "procfs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25485360a54d6861439d60facef26de713b1e126bf015ec8f98239467a2b82f7" +dependencies = [ + "bitflags", + "chrono", + "flate2", + "procfs-core", + "rustix", +] + +[[package]] +name = "procfs-core" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6401bf7b6af22f78b563665d15a22e9aef27775b79b149a66ca022468a4e405" +dependencies = [ + "bitflags", + "chrono", + "hex", +] + +[[package]] +name = "quote" +version = "1.0.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "scroll" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1257cd4248b4132760d6524d6dda4e053bc648c9070b960929bf50cfb1e7add" +dependencies = [ + "scroll_derive", +] + +[[package]] +name = "scroll_derive" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed76efe62313ab6610570951494bdaa81568026e0318eaa55f167de70eeea67d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a0c28ca5908dbdbcd52e6fdaa00358ab88637f8ab33e1f188dd510eb44b53d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +dependencies = [ + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + +[[package]] +name = "syn" +version = "2.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a26dbd934e5451d21ef060c018dae56fc073894c5a7896f882928a76e6d081b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "462eeb75aeb73aea900253ce739c8e18a67423fadf006037cd3ff27e82748a06" + +[[package]] +name = "uruntime" +version = "0.7.1" +dependencies = [ + "cfg-if", + "dotenv", + "fs2", + "goblin", + "memfd-exec", + "nix", + "num_cpus", + "procfs", + "sha2", + "signal-hook", + "tempfile", + "which", + "xxhash-rust", + "zstd", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "which" +version = "8.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3ef584124b911bcc3875c2f1472e80f24361ceb789bd1c62b3e9a3df9ff43c" +dependencies = [ + "libc", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 1f66bb6..908d62a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,8 @@ +cargo-features = ["panic-immediate-abort"] + [package] name = "uruntime" -version = "0.4.3" -readme = "README.md" +version = "0.7.1" license = "MIT" repository = "https://github.com/VHSgunzo/uruntime" description = "Universal RunImage and AppImage runtime with SquashFS and DwarFS supports" @@ -13,6 +14,7 @@ debug = false opt-level = "z" strip = "symbols" codegen-units = 1 +panic = "immediate-abort" [profile.dev] opt-level = 0 @@ -22,26 +24,36 @@ default = [ "dwarfs", "squashfs", ] -upx = [] lite = [] -dwarfs = [] +dwarfs = [ "dep:procfs", "dep:num_cpus" ] squashfs = [] appimage = [] [build-dependencies] -cfg-if = "1.0.0" -indexmap = "2.6.0" +fs2 = "0.4.3" +sha2 = "0.10.9" +tempfile = "3.23.0" zstd = { version = "0.13.3", default-features = false } +xxhash-rust = { version = "0.8.18", features = [ "xxh64" ] } [dependencies] -which = "7.0.0" -cfg-if = "1.0.0" -goblin = "0.9.0" -procfs = "0.17.0" -num_cpus = "1.16.0" -signal-hook = "0.3.17" +which = "8.0.5" +cfg-if = "1.0.4" +goblin = "0.10.7" +procfs = { version = "0.18.0", optional = true } +num_cpus = { version = "1.17.0", optional = true } +signal-hook = "0.4.4" zstd = { version = "0.13.3", default-features = false } -nix = { version = "0.30.1", features = [ "fs", "signal" ] } -xxhash-rust = { version = "0.8.15", features = [ "xxh3" ] } -dotenv = { git = "https://github.com/VHSgunzo/dotenv.git" } -memfd-exec = { git = "https://github.com/VHSgunzo/memfd-exec.git" } \ No newline at end of file +xxhash-rust = { version = "0.8.18", features = [ "xxh3" ] } +nix = { version = "0.31.3", features = [ "fs", "signal", "mount" ] } +dotenv = { git = "https://github.com/VHSgunzo/dotenv.git", rev = "367085af80a8d29d2241c85cbfc25736486230c5" } +memfd-exec = { git = "https://github.com/VHSgunzo/memfd-exec.git", rev = "2decf7d1cec3d183e55000526abd2e5bb6df40c5" } + +[target.'cfg(not(target_arch = "x86_64"))'.dependencies] +zstd = { version = "0.13.3", default-features = false, features = [ "no_asm" ] } + +[dev-dependencies] +fs2 = "0.4.3" +sha2 = "0.10.9" +tempfile = "3.23.0" +xxhash-rust = { version = "0.8.18", features = [ "xxh64" ] } diff --git a/Cross.toml b/Cross.toml deleted file mode 100644 index ea90136..0000000 --- a/Cross.toml +++ /dev/null @@ -1,2 +0,0 @@ -[target.aarch64-unknown-linux-musl] -dockerfile = "Dockerfile.aarch64" diff --git a/Dockerfile.aarch64 b/Dockerfile.aarch64 deleted file mode 100644 index b13b4d6..0000000 --- a/Dockerfile.aarch64 +++ /dev/null @@ -1,3 +0,0 @@ -FROM ghcr.io/cross-rs/aarch64-unknown-linux-musl:latest -ARG TARGETARCH -RUN curl -L https://github.com/upx/upx/releases/download/v5.0.0/upx-5.0.0-${TARGETARCH}_linux.tar.xz|tar -xJf - -C /usr/local/bin --strip-components=1 --wildcards "*/upx" diff --git a/LICENSE b/LICENSE index f74c4e3..bd1e3eb 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2025 VHSgunzo +Copyright (c) 2026 VHSgunzo Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index e5c7fb2..4c17002 100644 --- a/README.md +++ b/README.md @@ -1,270 +1,420 @@ -# URUNTIME -Universal [RunImage](https://github.com/VHSgunzo/runimage) and [AppImage](https://appimage.org/) runtime with [SquashFS](https://docs.kernel.org/filesystems/squashfs.html) and [DwarFS](https://github.com/mhx/dwarfs) supports +# uruntime -## To get started: -* **Download the latest revision** +`uruntime` is a static runtime for [RunImage](https://github.com/VHSgunzo/runimage) and [AppImage](https://appimage.org/). It detects an appended [SquashFS](https://docs.kernel.org/filesystems/squashfs.html) or [DwarFS](https://github.com/mhx/dwarfs) image, mounts it through FUSE, or extracts it and runs the application without FUSE. + +The project is intended for both users of prebuilt images and AppImage/RunImage authors. A single ELF contains the launcher, static filesystem tools, and mutable areas for configuration, environment variables, a signature, and update information. + +## How it works + +When started, `uruntime`: + +1. reads the runtime format, embedded settings, and boundary of the appended image; +2. identifies the filesystem by its signature and selects the embedded SquashFS or DwarFS helper; +3. prepares the `.env` file, portable directories, and internal environment variables; +4. creates user and mount namespaces with the requested UID/GID mapping when needed; +5. mounts the image through FUSE or falls back to extraction if the configuration permits it; +6. runs `AppRun` for an AppImage or runs `Run.sh` through the embedded `static/bash` for a RunImage; +7. after the application exits, removes the extracted directory or unmounts the image according to the reuse mode. + +Filesystem helpers are stored inside the runtime in Zstd-compressed form and executed through `memfd`. If `memfd-exec` is disabled, the runtime uses a temporary executable file. You can change the launch configuration, embedded environment, and update information in a finished runtime without recompiling it or rebuilding the appended image. + +## Features + +### Mounting and extraction + +- SquashFS and DwarFS in one runtime, or separate filesystem-specific variants. +- FUSE operation, forced extraction, and configurable fallback to extraction when FUSE is unavailable. +- A fallback limit based on the total file size: by default, automatic extraction is allowed only for files up to 350 MiB. +- Mount point reuse, including mount points created in a separate namespace. +- Delayed unmount after the mount is no longer in use: seconds, minutes, hours, or no time limit. +- An explicit mount or extraction directory and a separate FUSE debug mode. + +### DwarFS + +- Configurable worker count, cache size, block size, and readahead. +- Automatic reduction of the cache and worker count when available memory is low. +- Preloading of all blocks or the default `hotness` category. +- An `analysis_file` containing file access statistics. This profile can be passed to `mkdwarfs` the next time the image is built. +- A choice of `malloc` or `mmap` for block allocation. + +### Embedded tools and configuration + +- Direct invocation of `squashfuse`, `unsquashfs`, `sqfscat`, `mksquashfs`, `sqfstar`, `dwarfs`, `dwarfsck`, `mkdwarfs`, and `dwarfsextract` when the selected runtime variant includes the tool. +- Helper invocation through a runtime option or through a hard link, symbolic link, or runtime copy named after the tool. +- Updating embedded environment variables, update information, and signatures from a string or file. +- Loading variables from the embedded section and adjacent `${RUNTIME_NAME}.env` file; the `unset NAME` directive removes a variable. +- Four portable directories beside the image: home, data, config, and cache. + +### Isolation + +`unshare` creates user and mount namespaces and supports UID/GID mapping. This also allows mounting without a SUID `fusermount` on systems where user namespaces are available. + +`APPIMAGE_UNSHARE=2` and `RUNIMAGE_UNSHARE=2` enable `unshare` and drop capabilities before starting the main child process. The embedded `URUNTIME_UNSHARE=2` mode has the same behavior by default. Value `3` keeps normal mounting as the first choice and drops capabilities only when `unshare` is entered automatically as a fallback. Capabilities are dropped only after a namespace has been created or re-entered successfully, and only for the launched application; filesystem helpers retain the privileges needed for mounting. + +## Supported architectures + +| Artifact architecture | Rust target | Byte order | +|---|---|---| +| `x86_64` | `x86_64-unknown-linux-musl` | little-endian | +| `aarch64` | `aarch64-unknown-linux-musl` | little-endian | +| `riscv64` | `riscv64gc-unknown-linux-musl` | little-endian | +| `loongarch64` | `loongarch64-unknown-linux-musl` | little-endian | +| `ppc64` | `powerpc64-unknown-linux-musl` | big-endian | +| `ppc64le` | `powerpc64le-unknown-linux-musl` | little-endian | + +`ppc64` and `ppc64le` share the Rust `target_arch` value `powerpc64`, so runtimes and helpers are selected by the complete Rust target. Big-endian and little-endian artifacts are not interchangeable. + +All six targets are implemented in `build.rs`, `xtask`, and the CI matrix. Every release workflow builds and validates the complete 54-artifact matrix before publication. + +## Runtime variants + +For each architecture, `cargo xtask` generates nine variants. The complete filename is `uruntime--`. + +| Variant | Format | Filesystems | Contents | +|---|---|---|---| +| `runimage` | RunImage | SquashFS + DwarFS | full | +| `runimage-squashfs` | RunImage | SquashFS | full | +| `runimage-dwarfs` | RunImage | DwarFS | full | +| `appimage` | AppImage | SquashFS + DwarFS | full | +| `appimage-lite` | AppImage | SquashFS + DwarFS | no creation/checking tools | +| `appimage-squashfs` | AppImage | SquashFS | full | +| `appimage-squashfs-lite` | AppImage | SquashFS | no `mksquashfs` or `sqfstar` | +| `appimage-dwarfs` | AppImage | DwarFS | full | +| `appimage-dwarfs-lite` | AppImage | DwarFS | no `dwarfsck` or `mkdwarfs` | + +Lite variants retain the tools needed for mounting and extraction. Full variants can also create and check images. + +## Projects using uruntime + +- [AnyLinux-AppImages](https://github.com/pkgforge-dev/Anylinux-AppImages) — a collection of AppImage build scripts for many Linux applications. +- [GOverlay](https://github.com/benjamimgois/goverlay) — a graphical configurator for MangoHud, vkBasalt, and related gaming tools. +- [Ghostty AppImage](https://github.com/pkgforge-dev/ghostty-appimage) — a portable AppImage build of the Ghostty terminal. +- [Interstellar](https://github.com/interstellar-app/interstellar) — a client for Mbin, Lemmy, and PieFed. +- [QDiskInfo](https://github.com/edisionnano/QDiskInfo) — a graphical frontend for `smartctl` and drive SMART data. +- [CPU-X](https://github.com/TheTumultuousUnicornOfDarkness/CPU-X) — a system information and hardware monitoring application. +- [Eden](https://git.eden-emu.dev/eden-emu/eden) — a Nintendo Switch emulator. +- [PPSSPP](https://github.com/hrydgard/ppsspp) — a PlayStation Portable emulator. +- [RPCS3](https://github.com/RPCS3/rpcs3) — a PlayStation 3 emulator. +- [Converseen](https://github.com/Faster3ck/Converseen) — a batch image conversion and resizing application. +- [MangoJuice](https://github.com/radiolamp/mangojuice) — a graphical configuration tool for MangoHud. +- [RSS Guard](https://github.com/martinrotter/rssguard) — a desktop client for RSS, Atom, and other feed formats. + +## Getting a prebuilt runtime + +Prebuilt files are published on the [Releases](https://github.com/VHSgunzo/uruntime/releases) page. Choose a variant and architecture from the tables above, then make the file executable: + +```sh +chmod +x uruntime-appimage-x86_64 +./uruntime-appimage-x86_64 --appimage-help ``` -git clone https://github.com/VHSgunzo/uruntime.git && cd uruntime + +The option prefix depends on the format: + +- AppImage: `--appimage-*`, with `APPIMAGE_*` variables; +- RunImage: `--runtime-*`, with `RUNIMAGE_*` variables. + +In the reference below, `` means `appimage` or `runtime`, and `` means `APPIMAGE` or `RUNIMAGE`. + +## Usage + +### Main options + +| Option | Action | +|---|---| +| `---extract [PATTERN]` | Extract the image into the current directory; if a pattern is provided, extract only matching paths. | +| `---extract-and-run [ARGS]` | Extract the image and run the application without FUSE. | +| `---offset` | Print the byte offset where the filesystem image begins. | +| `---mount` | Mount the image, print the mount point, and wait for `Ctrl-C`. | +| `---unshare` | Try to create user and mount namespaces. | +| `---unshare-root` | Enable `unshare` and map the current user to UID 0 and GID 0. | +| `---unshare-uid UID` | Enable `unshare` and map the current UID to `UID`. The `--...-uid=UID` form is also accepted. | +| `---unshare-gid GID` | Enable `unshare` and map the current GID to `GID`. The `--...-gid=GID` form is also accepted. | +| `---unshare-drop-caps` | Enable `unshare` and drop capabilities before starting the application. | +| `---unshare-fallback-drop-caps` | Keep normal mounting as the first choice and drop capabilities if `unshare` is selected automatically as a fallback. | +| `---portable-home` | Create `${RUNTIME_NAME}.home`. | +| `---portable-share` | Create `${RUNTIME_NAME}.share`. | +| `---portable-config` | Create `${RUNTIME_NAME}.config`. | +| `---portable-cache` | Create `${RUNTIME_NAME}.cache`. | +| `---help` | Show help for the selected runtime. | +| `---version` | Print the runtime version. | +| `---signature` | Print the embedded digital signature. | +| `---addsign 'SIGN\|/file'` | Write a signature from an argument or file. | +| `---updateinfo` | Print update information. The full form `---updateinformation` is also accepted. | +| `---addupdinfo 'INFO\|/file'` | Write update information from an argument or file. | +| `---envs` | Print the embedded environment. | +| `---addenvs 'ENVS\|/file'` | Write the environment from an argument or file. | + +Examples: + +```sh +# Show the AppImage image offset +./My.AppImage --appimage-offset + +# Extract only matching files +./My.AppImage --appimage-extract 'usr/bin/*' + +# Run without FUSE +./My.AppImage --appimage-extract-and-run --help + +# Write update information to a finished image +./My.AppImage --appimage-addupdinfo \ + 'gh-releases-zsync|owner|project|latest|*.AppImage.zsync' + +# Embed an environment from a file +./My.AppImage --appimage-addenvs ./app.env ``` -* **Compile a binary** +The value for `addsign`, `addupdinfo`, or `addenvs` must fit in the preallocated ELF section. Current sizes are 1024 bytes for the signature and update information and 16 KiB for the environment. The command fills the unused remainder with zero bytes. + +### Embedded CLI tools + +| Option | Tool | Availability | +|---|---|---| +| `---squashfuse [ARGS]` | `squashfuse` | variants with SquashFS | +| `---unsquashfs [ARGS]` | `unsquashfs` | variants with SquashFS | +| `---sqfscat [ARGS]` | `sqfscat` | variants with SquashFS | +| `---mksquashfs [ARGS]` | `mksquashfs` | full variants with SquashFS | +| `---sqfstar [ARGS]` | `sqfstar` | full variants with SquashFS | +| `---dwarfs [ARGS]` | `dwarfs` | variants with DwarFS | +| `---dwarfsck [ARGS]` | `dwarfsck` | full variants with DwarFS | +| `---mkdwarfs [ARGS]` | `mkdwarfs` | full variants with DwarFS | +| `---dwarfsextract [ARGS]` | `dwarfsextract` | variants with DwarFS | + +The same tool can be invoked through the filename: + +```sh +ln uruntime-appimage-x86_64 mksquashfs +./mksquashfs --help ``` -rustup toolchain add nightly -rustup target add x86_64-unknown-linux-musl -rustup component add rust-src --toolchain nightly - -cargo xtask -# Tasks: -# x86_64 build x86_64 RunImage and AppImage uruntime -# runimage-x86_64 build x86_64 RunImage uruntime -# appimage-x86_64 build x86_64 AppImage uruntime -# appimage-lite-x86_64 build x86_64 AppImage uruntime (no dwarfsck, mkdwarfs, mksquashfs, sqfstar) -# appimage-squashfs-x86_64 build x86_64 AppImage uruntime (SquashFS-only) -# appimage-squashfs-lite-x86_64 build x86_64 AppImage uruntime (SquashFS-only no mksquashfs, sqfstar) -# appimage-dwarfs-x86_64 build x86_64 AppImage uruntime (DwarFS-only) -# appimage-dwarfs-lite-x86_64 build x86_64 AppImage uruntime (DwarFS-only no dwarfsck, mkdwarfs) -# -# aarch64 build aarch64 RunImage and AppImage uruntime -# runimage-aarch64 build aarch64 RunImage uruntime -# appimage-aarch64 build aarch64 AppImage uruntime -# appimage-lite-aarch64 build aarch64 AppImage uruntime (no dwarfsck, mkdwarfs, mksquashfs, sqfstar) -# appimage-squashfs-aarch64 build aarch64 AppImage uruntime (SquashFS-only) -# appimage-squashfs-lite-aarch64 build aarch64 AppImage uruntime (SquashFS-only no mksquashfs, sqfstar) -# appimage-dwarfs-aarch64 build aarch64 AppImage uruntime (DwarFS-only) -# appimage-dwarfs-lite-aarch64 build aarch64 AppImage uruntime (DwarFS-only no dwarfsck, mkdwarfs) -# -# all build all of the above - -# for RunImage x86_64 -cargo xtask runimage-x86_64 - -# for AppImage x86_64 -cargo xtask appimage-x86_64 + +### Portable directories + +If the following directories exist beside the image, the runtime changes the corresponding variables before starting the application: + +| Directory | Variable | +|---|---| +| `${RUNTIME_NAME}.home` | `HOME` | +| `${RUNTIME_NAME}.share` | `XDG_DATA_HOME` | +| `${RUNTIME_NAME}.config` | `XDG_CONFIG_HOME` | +| `${RUNTIME_NAME}.cache` | `XDG_CACHE_HOME` | + +You can create them with the matching `---portable-*` options. The directories are tied to the file's current name and location. + +## Local builds + +The repository uses nightly Rust from `rust-toolchain.toml` and Cargo `build-std`. Install Rust and the `rust-src` component. Preparing helpers and publishing files to `dist/` also requires `curl` and `llvm-objcopy`; automatic Zig installation requires `tar` with XZ support. + +```sh +git clone https://github.com/VHSgunzo/uruntime.git +cd uruntime +rustup component add rust-src + +# List targets and all 54 tasks +cargo xtask help + +# Run all local checks for the current platform's musl target +cargo xtask check + +# Do the same for an explicitly selected supported Rust target +cargo xtask check x86_64-unknown-linux-musl + +# Build nine variants for an architecture +cargo xtask x86_64 +cargo xtask aarch64 + +# Build one variant +cargo xtask appimage-squashfs-riscv64 + +# Build the full matrix: 6 architectures x 9 variants +cargo xtask all ``` -See [Build step in ci.yml](https://github.com/VHSgunzo/uruntime/blob/main/.github/workflows/ci.yml#L34) -* Or take an already precompiled from the [releases](https://github.com/VHSgunzo/uruntime/releases) +Each successful task creates `dist/uruntime--`. + +`cargo xtask check` detects the musl target for the current Linux platform. For example, it uses `x86_64-unknown-linux-musl` on `x86_64` and `aarch64-unknown-linux-musl` on `aarch64`. The command runs `cargo fmt --check`, Check, Clippy with `-D warnings`, tests for the root package and `xtask`, validation of all checksums, and `git diff --check`. Root Check, Clippy, and tests always use the same pinned Zig linker backend as release builds, including when the selected musl target matches the host architecture. One of the six Rust targets can be passed explicitly as a second argument; a foreign target additionally requires the matching QEMU user-mode executable in `PATH` to run root tests. The `xtask` package itself is always checked and tested natively. + +Cargo orchestrates every release build, while the project-provided Zig linker wrapper links both native and foreign musl targets consistently. The separate `cross` backend is no longer used. The pinned Zig 0.16.0 index, current-host archive, and extracted installation are cached under `target/toolchains/`. Archives and installations are keyed by version, host, and full SHA-256; every reuse is verified before use. Automatic downloads are supported on Linux hosts with `x86_64`, `aarch64`, `riscv64`, `loongarch64`, or `powerpc64le`. On another host, set `URUNTIME_ZIG`; running `zig version` for the selected file must return exactly `0.16.0`. + +QEMU is not involved in the build and is not needed to extract DwarFS helpers. It is used only to run `--version` on finished foreign-architecture files during smoke tests. + +`build.rs` pins DwarFS 0.15.7, squashfs-tools 4.7.5.r2, and squashfuse 0.6.3.r2. Normal builds do not depend on UPX. DwarFS publishes Zstd self-extracting wrappers; `build.rs` parses the `SQUEEZE!` trailer, checks the size and XXH64, extracts the target ELF on the host, and then verifies the SHA-256 and ELF machine/endian. The foreign ELF is never executed during this process. + +## Verifying and updating checksums -### **Built-in configuration:** -You can change the startup logic by changing the built-in uruntime parameters. -* `URUNTIME_EXTRACT` - Specifies the logic of extracting or mounting +`checksums.txt` pins two checksums for every filesystem helper: the downloaded file and the extracted payload. It also records the URLs and SHA-256 values of Zig archives for supported Linux hosts. + +```sh +# Download the sources, recompute the data, and check for drift +cargo xtask update-checksums --check + +# Update checksums.txt after an intentional version or URL change +cargo xtask update-checksums + +git diff -- checksums.txt ``` -# URUNTIME_EXTRACT=0 - FUSE mounting only -sed -i 's|URUNTIME_EXTRACT=[0-9]|URUNTIME_EXTRACT=0|' /path/uruntime -# URUNTIME_EXTRACT=1 - Do not use FUSE mounting, but extract and run -sed -i 's|URUNTIME_EXTRACT=[0-9]|URUNTIME_EXTRACT=1|' /path/uruntime +Both commands check all 30 helper sources, not just the current architecture alone. A source is reused only when its bounded SHA-256 matches the current manifest; a missing or mismatched source is downloaded atomically into the same dependency-specific cache later consumed by `build.rs`. SquashFUSE, squashfs-tools, and DwarFS sources have separate architecture/version cache trees, so updating one dependency does not invalidate the others. The Zig index and current-host archive use the same verified-cache policy. A fully populated cache allows checksum validation without network access. Always review the diff after an update. `URUNTIME_CURL=/path/to/curl` selects the download program for both `build.rs` and `update-checksums`. + +## CI builds + +The `.github/workflows/ci.yml` workflow has three parts: -# URUNTIME_EXTRACT=2 - Try to use FUSE mounting and if it is unavailable extract and run -sed -i 's|URUNTIME_EXTRACT=[0-9]|URUNTIME_EXTRACT=2|' /path/uruntime +1. Preflight runs the canonical `cargo xtask check` command: formatting, Check, Clippy, Rust tests, `xtask` tests, pinned helper/Zig validation, and `git diff --check`. +2. Six independent build jobs run `cargo xtask `, verify the exact list of nine files, ELF64 machine/endian, absence of `PT_INTERP` and `DT_NEEDED`, required sections, and runtime magic; foreign jobs run `--version` through QEMU. +3. On a tag push, the release job builds exactly 54 files and publishes them through a verified draft stage. A rerun can safely refresh the release for the same tag: old assets are deleted only after the release becomes a draft, and it is made public again only after the new complete manifest has been verified. -# URUNTIME_EXTRACT=3 - As above, but if the image size is less than 350 MB (default) -sed -i 's|URUNTIME_EXTRACT=[0-9]|URUNTIME_EXTRACT=3|' /path/uruntime +A QEMU smoke test does not replace a FUSE mount test on a foreign architecture. Full mount/run behavior must be tested separately where the runner provides a working `/dev/fuse`. + +[RELEASING.md](RELEASING.md) describes the step-by-step process for updating the code, Rust dependencies, helpers, and Zig, and for creating or reissuing a tag. + +## Embedded configuration + +Four strings are stored directly in the runtime ELF. In the current build they are `URUNTIME_MOUNT=3`, `URUNTIME_CLEANUP=1`, `URUNTIME_EXTRACT=3`, and `URUNTIME_UNSHARE=0`. They can be replaced in a finished runtime, including after an image has been appended. The value must remain a single digit so that the file size and image offset do not change. + +### `URUNTIME_EXTRACT` + +| Value | Behavior | +|---|---| +| `0` | Use only a FUSE mount; do not extract automatically. | +| `1` | Always extract and run without FUSE. | +| `2` | Try FUSE first; on failure, extract regardless of file size. | +| `3` | Try FUSE first; on failure, extract only if the file is no larger than 350 MiB. This is the default. | + +```sh +sed -i 's|URUNTIME_EXTRACT=[0-9]|URUNTIME_EXTRACT=2|' /path/to/runtime ``` -* `URUNTIME_CLEANUP` - Specifies the logic of cleanup after extract and run +The explicit `---mount` option never falls back to extraction. The `---extract` option works independently of the fallback mode. + +### `URUNTIME_CLEANUP` + +| Value | Behavior | +|---|---| +| `0` | Do not remove the directory after extract-and-run. | +| `1` | Remove the extracted directory after the application exits and the wait period ends. This is the default. | + +```sh +sed -i 's|URUNTIME_CLEANUP=[0-9]|URUNTIME_CLEANUP=0|' /path/to/runtime ``` -# URUNTIME_CLEANUP=0 - Disable extracting directory cleanup -sed -i 's|URUNTIME_CLEANUP=[0-9]|URUNTIME_CLEANUP=0|' /path/uruntime -# URUNTIME_CLEANUP=1 - Enable extracting directory cleanup (default) -sed -i 's|URUNTIME_CLEANUP=[0-9]|URUNTIME_CLEANUP=1|' /path/uruntime +`NO_CLEANUP=1` overrides cleanup for one run in extraction mode. + +### `URUNTIME_UNSHARE` + +| Value | Behavior | +|---|---| +| `0` | Do not enable `unshare` in advance. The runtime may still try it if FUSE is unavailable without a SUID `fusermount`. This is the default. | +| `1` | Create user and mount namespaces by default. | +| `2` | Create namespaces and drop capabilities before starting the application. | +| `3` | Do not enable `unshare` in advance. If the runtime enters `unshare` automatically as a fallback, drop capabilities before starting the application. Explicit `---unshare` and `_UNSHARE=1` requests do not enable capability dropping by themselves. | + +```sh +sed -i 's|URUNTIME_UNSHARE=[0-9]|URUNTIME_UNSHARE=2|' /path/to/runtime ``` -* `URUNTIME_MOUNT` - Specifies the mount logic +Use mode `3` when normal FUSE mounting should remain the first attempt, but an automatically selected `unshare` fallback must launch the application without ambient, bounding, effective, permitted, or inheritable capabilities: + +```sh +sed -i 's|URUNTIME_UNSHARE=[0-9]|URUNTIME_UNSHARE=3|' /path/to/runtime ``` -# URUNTIME_MOUNT=0 - Reuse mount point and disable unmounting of the mount directory -sed -i 's|URUNTIME_MOUNT=[0-9]|URUNTIME_MOUNT=0|' /path/uruntime -# URUNTIME_MOUNT=1 - Random mount points and unmounting of the mount directory -sed -i 's|URUNTIME_MOUNT=[0-9]|URUNTIME_MOUNT=1|' /path/uruntime +### `URUNTIME_MOUNT` -# URUNTIME_MOUNT=2 - Reuse mount point and unmounting of the mount directory -# with a 30 minutes delay of inactivity -sed -i 's|URUNTIME_MOUNT=[0-9]|URUNTIME_MOUNT=2|' /path/uruntime +| Value | Behavior | +|---|---| +| `0` | Reuse a stable mount point; by default, the FUSE mount remains mounted indefinitely. | +| `1` | Use a random mount point and unmount after the application exits. | +| `2` | Use a stable mount point and unmount after 30 minutes without use. | +| `3` | Use a stable mount point and unmount after 5 seconds without use. This is the default. | -# URUNTIME_MOUNT=3 - Reuse mount point and unmounting of the mount directory -# with a 5 second delay of inactivity (default) -sed -i 's|URUNTIME_MOUNT=[0-9]|URUNTIME_MOUNT=3|' /path/uruntime +```sh +sed -i 's|URUNTIME_MOUNT=[0-9]|URUNTIME_MOUNT=1|' /path/to/runtime ``` -
-RunImage runtime usage - +`REUSE_CHECK_DELAY` changes the delay for reuse modes. `NO_UNMOUNT=1` keeps the mount indefinitely for one run. +## Environment variables + +### Paths and launch mode + +| Variable | Value | +|---|---| +| `URUNTIME` | Path to the executable runtime processing the image. The runtime sets this variable itself. | +| `URUNTIME_DIR` | Directory containing this runtime. The runtime sets this variable itself. | +| `_EXTRACT_AND_RUN=1` | Extract and run without FUSE. | +| `NO_CLEANUP=1` | Do not remove data after extract-and-run. | +| `NO_UNMOUNT=1` | Do not unmount the image after the application exits; enables mount point reuse. | +| `TMPDIR=/path` | Base temporary directory for mounting or extraction. | +| `_TARGET_DIR=/path` | Exact directory for mounting or extraction. | +| `REUSE_CHECK_DELAY=5s` | Delay before checking whether the directory is in use. Accepts an integer number of seconds or one `s`, `m`, or `h` suffix; `inf` disables the timeout, while `0` disables reuse. An invalid value produces a one-second delay. | +| `FUSERMOUNT_PROG=/path` | Explicit path to a SUID `fusermount`/`fusermount3`. | +| `ENABLE_FUSE_DEBUG=1` | Enable debug output from the selected FUSE helper. | +| `TARGET_=/path` | Perform a maintenance operation on the specified AppImage/RunImage instead of the runtime itself. | +| `NO_MEMFDEXEC=1` | Run the extracted helper through a temporary file instead of `memfd-exec`. | + +AppImage uses `APPIMAGE_EXTRACT_AND_RUN`, `APPIMAGE_TARGET_DIR`, and `TARGET_APPIMAGE`. RunImage uses `RUNIMAGE_EXTRACT_AND_RUN`, `RUNIMAGE_TARGET_DIR`, and `TARGET_RUNIMAGE`. + +### `unshare` and UID/GID mapping + +| Variable | Value | +|---|---| +| `_UNSHARE=1` | Create user and mount namespaces. | +| `_UNSHARE=2` | Create namespaces and drop ambient, bounding, effective, permitted, and inheritable capabilities before starting the application. | +| `_UNSHARE=3` | Do not enable `unshare` in advance; drop capabilities if it is selected automatically as a fallback. | +| `_UNSHARE_ROOT=1` | Map the current user to UID 0 and GID 0 inside the user namespace. | +| `_UNSHARE_UID=` | Map the current UID to the specified UID inside the namespace. | +| `_UNSHARE_GID=` | Map the current GID to the specified GID inside the namespace. | + + +Substituting `` gives the complete set of `APPIMAGE_*` or `RUNIMAGE_*` variables. Any UID/GID mapping also enables `unshare`. If `*_UNSHARE_ROOT=1` or `---unshare-root` is set, root mapping takes precedence over separate UID/GID values from either CLI options or environment variables. + +The CLI options can be combined. For example: + +```sh +./My.AppImage \ + --appimage-unshare-uid 1000 \ + --appimage-unshare-gid 1000 \ + --appimage-unshare-drop-caps ``` - Runtime options: - --runtime-extract [PATTERN] Extract content from embedded filesystem image - If pattern is passed, only extract matching files - --runtime-extract-and-run [ARGS] Run the RunImage afer extraction without using FUSE - --runtime-offset Print byte offset to start of embedded filesystem image - --runtime-portable-home Create a portable home folder to use as $HOME - --runtime-portable-share Create a portable share folder to use as $XDG_DATA_HOME - --runtime-portable-config Create a portable config folder to use as $XDG_CONFIG_HOME - --runtime-help Print this help - --runtime-version Print version of Runtime - --runtime-signature Print digital signature embedded in RunImage - --runtime-addsign 'SIGN|/file' Add digital signature to RunImage - --runtime-updateinfo[rmation] Print update info embedded in RunImage - --runtime-addupdinfo 'INFO|/file' Add update info to RunImage - --runtime-envs Print environment variables embedded in RunImage - --runtime-addenvs 'ENVS|/file' Add environment variables to RunImage - --runtime-mount Mount embedded filesystem image and print - mount point and wait for kill with Ctrl-C - - Embedded tools options: - --runtime-squashfuse [ARGS] Launch squashfuse - --runtime-unsquashfs [ARGS] Launch unsquashfs - --runtime-sqfscat [ARGS] Launch sqfscat - --runtime-mksquashfs [ARGS] Launch mksquashfs - --runtime-sqfstar [ARGS] Launch sqfstar - --runtime-dwarfs [ARGS] Launch dwarfs - --runtime-dwarfsck [ARGS] Launch dwarfsck - --runtime-mkdwarfs [ARGS] Launch mkdwarfs - --runtime-dwarfsextract [ARGS] Launch dwarfsextract - - Also you can create a hardlink, symlink or rename the runtime with - the name of the built-in utility to use it directly. - - Portable home and config: - - If you would like the application contained inside this RunImage to store its - data alongside this RunImage rather than in your home directory, then you can - place a directory named - - for portable-home: - "${RUNTIME_NAME}.home" - - for portable-share: - "${RUNTIME_NAME}.share" - - for portable-config: - "${RUNTIME_NAME}.config" - - Or you can invoke this RunImage with the --runtime-portable-home or - --runtime-portable-share or --runtime-portable-config option, - which will create this directory for you. - As long as the directory exists and is neither moved nor renamed, the - application contained inside this RunImage to store its data in this - directory rather than in your home directory - - Environment variables: - - URUNTIME Path to uruntime - URUNTIME_DIR Path to uruntime directory - RUNTIME_EXTRACT_AND_RUN=1 Run the RunImage afer extraction without using FUSE - NO_CLEANUP=1 Do not clear the unpacking directory after closing when - using extract and run option for reuse extracted data - NO_UNMOUNT=1 Do not unmount the mount directory after closing - for reuse mount point - TMPDIR=/path Specifies a custom path for mounting or extracting the image - URUNTIME_TARGET_DIR=/path Specifies the exact path for mounting or extracting the image - REUSE_CHECK_DELAY=5s Specifies the delay between checks of using the image dir (inf|1|1s|1m|1h) - FUSERMOUNT_PROG=/path Specifies a custom path for fusermount - ENABLE_FUSE_DEBUG=1 Enables debug mode for the mounted filesystem - TARGET_RUNIMAGE=/path Operate on a target RunImage rather than this file itself - NO_MEMFDEXEC=1 Do not use memfd-exec (use a temporary file instead) - DWARFS_WORKERS=2 Number of worker threads for DwarFS (default: equal CPU threads) - DWARFS_CACHESIZE=1024M Size of the block cache, in bytes for DwarFS (suffixes K, M, G) - DWARFS_BLOCKSIZE=512K Size of the block file I/O, in bytes for DwarFS (suffixes K, M, G) - DWARFS_READAHEAD=32M Set readahead size, in bytes for DwarFS (suffixes K, M, G) - DWARFS_PRELOAD_ALL=1 Enable preloading of all blocks from the DwarFS file system - DWARFS_ANALYSIS_FILE=/path A file for profiling open files when launching the application for DwarFS - DWARFS_USE_MMAP=1 Use mmap for allocating blocks for DwarFS - - Environment variables can be specified in the env file (see https://crates.io/crates/dotenv) - and environment variables can also be deleted using `unset ENV_VAR` in the end of the env file: - "${RUNTIME_NAME}.env" - You can also embed environment variables directly into runtime using the --runtime-addenvs option. -``` -
+When an ordinary `unshare` request and a capability-drop mode are both present, capability dropping takes precedence. Use `--` to stop runtime option parsing and pass the following arguments unchanged to the application; the separator itself is consumed by the runtime. + +### DwarFS settings + +| Variable | Value | +|---|---| +| `DWARFS_WORKERS=2` | Explicit number of worker threads. Without this variable, the runtime selects the count based on cache size and CPU count. | +| `DWARFS_CACHESIZE=1024M` | Block cache size. The `K`, `M`, and `G` suffixes are supported. Without this variable, the size is selected from available memory; 1024M is the fallback if `/proc` is unavailable. | +| `DWARFS_BLOCKSIZE=512K` | Block I/O size; the default is 512K. | +| `DWARFS_READAHEAD=32M` | Readahead size; the default is 32M. | +| `DWARFS_PRELOAD_ALL=1` | Preload all blocks; without this variable, `preload_category=hotness` is used. | +| `DWARFS_ANALYSIS_FILE=/path` | Write a profile of opened files to the specified file. | +| `DWARFS_USE_MMAP=1` | Use the `mmap` block allocator; without this variable, `malloc` is used. | -
-AppImage runtime usage - +### The `.env` file and embedded environment +The runtime processes the embedded `.envs` section first and then the adjacent file: + +```text +${RUNTIME_NAME}.env ``` - Runtime options: - --appimage-extract [PATTERN] Extract content from embedded filesystem image - If pattern is passed, only extract matching files - --appimage-extract-and-run [ARGS] Run the AppImage afer extraction without using FUSE - --appimage-offset Print byte offset to start of embedded filesystem image - --appimage-portable-home Create a portable home folder to use as $HOME - --appimage-portable-share Create a portable share folder to use as $XDG_DATA_HOME - --appimage-portable-config Create a portable config folder to use as $XDG_CONFIG_HOME - --appimage-help Print this help - --appimage-version Print version of Runtime - --appimage-signature Print digital signature embedded in AppImage - --appimage-addsign 'SIGN|/file' Add digital signature to AppImage - --appimage-updateinfo[rmation] Print update info embedded in AppImage - --appimage-addupdinfo 'INFO|/file' Add update info to AppImage - --appimage-envs Print environment variables embedded in AppImage - --appimage-addenvs 'ENVS|/file' Add environment variables to AppImage - --appimage-mount Mount embedded filesystem image and print - mount point and wait for kill with Ctrl-C - - Embedded tools options: - --appimage-squashfuse [ARGS] Launch squashfuse - --appimage-unsquashfs [ARGS] Launch unsquashfs - --appimage-sqfscat [ARGS] Launch sqfscat - --appimage-mksquashfs [ARGS] Launch mksquashfs - --appimage-sqfstar [ARGS] Launch sqfstar - --appimage-dwarfs [ARGS] Launch dwarfs - --appimage-dwarfsck [ARGS] Launch dwarfsck - --appimage-mkdwarfs [ARGS] Launch mkdwarfs - --appimage-dwarfsextract [ARGS] Launch dwarfsextract - - Also you can create a hardlink, symlink or rename the runtime with - the name of the built-in utility to use it directly. - - Portable home and config: - - If you would like the application contained inside this AppImage to store its - data alongside this AppImage rather than in your home directory, then you can - place a directory named - - for portable-home: - "${RUNTIME_NAME}.home" - - for portable-config: - "${RUNTIME_NAME}.config" - - Or you can invoke this AppImage with the --appimage-portable-home or - --appimage-portable-share or --appimage-portable-config option, - which will create this directory for you. - As long as the directory exists and is neither moved nor renamed, the - application contained inside this AppImage to store its data in this - directory rather than in your home directory - - Environment variables: - - URUNTIME Path to uruntime - URUNTIME_DIR Path to uruntime directory - APPIMAGE_EXTRACT_AND_RUN=1 Run the AppImage afer extraction without using FUSE - NO_CLEANUP=1 Do not clear the unpacking directory after closing when - using extract and run option for reuse extracted data - NO_UNMOUNT=1 Do not unmount the mount directory after closing - for reuse mount point - TMPDIR=/path Specifies a custom path for mounting or extracting the image - URUNTIME_TARGET_DIR=/path Specifies the exact path for mounting or extracting the image - REUSE_CHECK_DELAY=5s Specifies the delay between checks of using the image dir (inf|1|1s|1m|1h) - FUSERMOUNT_PROG=/path Specifies a custom path for fusermount - ENABLE_FUSE_DEBUG=1 Enables debug mode for the mounted filesystem - TARGET_APPIMAGE=/path Operate on a target AppImage rather than this file itself - NO_MEMFDEXEC=1 Do not use memfd-exec (use a temporary file instead) - DWARFS_WORKERS=2 Number of worker threads for DwarFS (default: equal CPU threads) - DWARFS_CACHESIZE=1024M Size of the block cache, in bytes for DwarFS (suffixes K, M, G) - DWARFS_BLOCKSIZE=512K Size of the block file I/O, in bytes for DwarFS (suffixes K, M, G) - DWARFS_READAHEAD=32M Set readahead size, in bytes for DwarFS (suffixes K, M, G) - DWARFS_PRELOAD_ALL=1 Enable preloading of all blocks from the DwarFS file system - DWARFS_ANALYSIS_FILE=/path A file for profiling open files when launching the application for DwarFS - DWARFS_USE_MMAP=1 Use mmap for allocating blocks for DwarFS - - Environment variables can be specified in the env file (see https://crates.io/crates/dotenv) - and environment variables can also be deleted using `unset ENV_VAR` in the end of the env file: - "${RUNTIME_NAME}.env" - You can also embed environment variables directly into runtime using the --appimage-addenvs option. + +For example, `/opt/My.AppImage` uses `/opt/My.AppImage.env`. Variable syntax follows the project's [`dotenv`](https://github.com/VHSgunzo/dotenv) fork. After each source is read, lines in the form `unset NAME` remove the specified variables: + +```dotenv +QT_QPA_PLATFORM=xcb +APP_DEBUG=1 +unset LD_PRELOAD ``` -
+Embed such a file with: + +```sh +./My.AppImage --appimage-addenvs ./My.AppImage.env +``` + +The external file is useful for local changes, while the embedded section travels with the image. + +## License + +[MIT](LICENSE) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md new file mode 100644 index 0000000..e1a16c2 --- /dev/null +++ b/RELEASE_NOTES.md @@ -0,0 +1,22 @@ +## uruntime v0.7.1 + +### Faster startup + +- Eliminated per-launch copies of embedded filesystem helpers; the runtime now uses their static embedded bytes directly. +- Reads and validates the ELF prefix sequentially without zero-filling large buffers or rereading the same ranges. +- Reduced temporary strings, argument cloning, metadata lookups, and permission probes in the startup path. +- Kept exact filesystem magic checks and all malformed-ELF range, endian, and size validation. +- Improved both normal startup and the default reusable-mount path without changing mount or extraction behavior. + +### Reproducible cache validation + +- Checksum validation and normal builds now consume the same verified local helper-source cache. +- Missing or mismatched helper releases are downloaded atomically; valid cached releases are reused. +- SquashFUSE, squashfs-tools, and DwarFS sources are cached independently by architecture and dependency version. +- The Zig release index, current-host archive, and extracted installation are cached and verified. +- Zig archives and installations are content-addressed by version, host, and full SHA-256 to prevent races between verification and use. + +### Validation + +- Preserves all six architectures and all nine RunImage/AppImage variants per architecture. +- Covers little-endian and big-endian ELF layouts, exact SquashFS/DwarFS boundaries, unshare option handling, and reusable mount behavior. diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..6d17a8f --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,227 @@ +# Updating and releasing uruntime + +This document covers a normal version release, updates to Rust dependencies, filesystem helpers, and Zig, pre-release validation through the `action` branch, and safe reissuing from the same tag. + +## 1. Prepare a working branch + +Start from an up-to-date `main` and create a separate branch: + +```sh +git switch main +git pull --ff-only +git switch -c release/v0.7.1 +``` + +Before making changes, confirm that the working tree contains no accidental files: + +```sh +git status --short +git diff --check +``` + +Build directories, local helper caches, CI staging, editor settings, and Python caches are excluded through `.gitignore`. Both lockfiles, `Cargo.lock` and `xtask/Cargo.lock`, must be tracked by Git. + +## 2. Make the changes + +### uruntime code only + +Change the code and update the version in the root `Cargo.toml`: + +```toml +[package] +version = "0.7.1" +``` + +`src/main.rs` gets the version from `CARGO_PKG_VERSION`; there is no separate constant to update. + +### Rust dependencies + +Change the constraints or pinned `rev` values in the relevant `Cargo.toml`, then update the corresponding lockfile: + +```sh +# Root package +cargo update + +# Separate xtask package, if its dependencies changed +cargo update --manifest-path xtask/Cargo.toml + +git diff -- Cargo.toml Cargo.lock xtask/Cargo.toml xtask/Cargo.lock +``` + +Do not edit lockfiles manually. Normal builds and CI use `--locked`, so an unrecorded dependency graph change will be caught before the release matrix runs. + +### SquashFS or DwarFS helpers + +First release and verify the new assets in the corresponding upstream/helper repository. Change the relevant version in `uruntime/build_support.rs`: + +```rust +pub const DWARFS_VERSION: &str = "..."; +pub const SQUASHFS_TOOLS_VERSION: &str = "..."; +pub const SQUASHFUSE_VERSION: &str = "..."; +``` + +Then regenerate the manifest: + +```sh +cargo xtask update-checksums +git diff -- checksums.txt +``` + +The command validates all 30 helper sources, reusing only local release files whose SHA-256 matches the current manifest. Missing or mismatched files are downloaded atomically into dependency-specific architecture/version caches that are also consumed by `build.rs`. It validates helper boundaries and ELF types, extracts DwarFS payloads, and records the SHA-256 of both each source file and its final payload. Review the `checksums.txt` diff before committing. + +### Zig + +Change `ZIG_VERSION` in `build_support.rs`, then run: + +```sh +cargo xtask update-checksums +git diff -- checksums.txt +``` + +`xtask` reads the official `https://ziglang.org/download/index.json` and updates the URLs and SHA-256 values of the archives for supported Linux hosts. The index, current-host archive, and extracted installation are cached under `target/toolchains/`; archives and installations are keyed by version, host, and full SHA-256. Checksum validation and real builds reuse the same verified archive. + +## 3. Run local checks + +Update `RELEASE_NOTES.md` with the public notes for the version being prepared. The release workflow uses this file verbatim when creating or refreshing the GitHub release, so same-tag reruns do not replace the intended notes with an automation placeholder. + +Run the full check suite before pushing: + +```sh +cargo xtask check +``` + +By default, `xtask` selects the musl target for the current Linux platform. On `x86_64`, this is `x86_64-unknown-linux-musl`. You can specify the target explicitly: + +```sh +cargo xtask check x86_64-unknown-linux-musl +``` + +Root Check, Clippy, and tests always use the pinned Zig linker backend, including when the selected musl target matches the host architecture. An explicitly selected foreign target additionally requires the matching QEMU user-mode executable (for example, `qemu-aarch64` or `qemu-aarch64-static`) in `PATH` to run the root tests. + +The command runs `cargo fmt --check`, root Check/Clippy/tests with `--locked`, separate Check/Clippy/tests for `xtask`, `cargo xtask update-checksums --check`, and `git diff --check` in sequence. It stops at the first failure. The checksum step validates all pinned helper sources and Zig metadata. It uses the network only for missing, mismatched, or stale cache entries; a fully populated verified cache works offline. + +If cross-linking changed, also build one foreign runtime: + +```sh +cargo xtask runimage-aarch64 +qemu-aarch64 dist/uruntime-runimage-aarch64 --runtime-version +``` + +QEMU is needed only for the smoke test of the finished foreign ELF. It does not participate in the build. + +## 4. Commit every reproducible input + +Confirm that Git sees the source files, CI scripts, manifests, and both lockfiles, but not build/cache directories: + +```sh +git status --short --untracked-files=all +git check-ignore -v dist target xtask/target assets-x86_64 || true +if git check-ignore -q Cargo.lock || git check-ignore -q xtask/Cargo.lock; then + echo 'Cargo.lock is still ignored' >&2 + exit 1 +fi +``` + +After checking, create the commit. For example: + +```sh +git add -A +git diff --cached --check +git commit -m "Release v0.7.1" +``` + +Do not use `git add -f` for files under `dist/`, `target/`, `assets-*`, or `release-dist/`. + +## 5. Validate the full matrix without releasing + +The workflow runs when you push to the `action` branch: + +```sh +git push origin HEAD:action +``` + +This run performs preflight, builds all six architectures, verifies nine files per architecture, and runs foreign runtimes through QEMU. It does not create a GitHub Release for the `action` branch. + +Monitor the run through the GitHub UI or `gh`: + +```sh +gh run list --branch action --limit 5 +gh run watch +``` + +After the matrix passes, transfer the exact verified commit to `main` using the project's chosen method, then push it: + +```sh +git switch main +git pull --ff-only +# Merge/cherry-pick the verified commit +git push origin main +``` + +## 6. Create a new release + +Confirm that `HEAD` contains the intended version and the code that passed CI: + +```sh +git status --short +git log -1 --oneline +grep '^version = ' Cargo.toml +``` + +Create and push an annotated tag: + +```sh +git tag -a v0.7.1 -m "uruntime v0.7.1" +git push origin v0.7.1 +``` + +The tag-push workflow rebuilds 54 files from the tagged commit. The release job: + +1. verifies that the remote tag points to `github.sha`; +2. creates a new draft or changes the existing release for that tag back to a draft; +3. deletes the old assets from that release; +4. uploads exactly 54 freshly verified runtimes; +5. reads the complete paginated asset manifest by numeric release ID; +6. verifies the remote tag SHA again; +7. publishes the same numeric release ID. + +If upload or verification is interrupted, the release remains a draft and does not publicly expose a mixture of old and new assets. + +## 7. Reissue from the same tag + +### The tag and commit have not changed + +Click **Re-run all jobs** on the original tag-push workflow, or run: + +```sh +gh run rerun +``` + +CI rebuilds the tagged commit and safely replaces the release assets through the draft stage. + +### The tag has moved to another commit + +Move the local annotated tag and push the tag update itself: + +```sh +git tag -fa v0.7.1 -m "uruntime v0.7.1" +git push --force origin refs/tags/v0.7.1 +``` + +The force-push creates a new tag-push workflow with a new `github.sha`. Do not rerun the old workflow for the previous commit: its SHA check must fail after the tag moves. + +Moving a published tag breaks users', package managers', and caches' expectation that releases are immutable. Do this only for a deliberate release correction. For a normal fix, prefer a new version number such as `v0.7.2`. + +## 8. Verify the published release + +```sh +release_id=$(gh api repos/VHSgunzo/uruntime/releases/tags/v0.7.1 --jq .id) + +gh api --paginate --slurp \ + "repos/VHSgunzo/uruntime/releases/$release_id/assets?per_page=100" \ + --jq 'add | length' + +gh release view v0.7.1 +``` + +The expected asset count is `54`. CI performs the same check automatically and publishes the release only after the complete manifest matches. diff --git a/build.rs b/build.rs index 4de6b59..acdf307 100644 --- a/build.rs +++ b/build.rs @@ -1,115 +1,57 @@ -use std::{ - env, - path::Path, - process::{exit, Command}, - os::unix::fs::{symlink, PermissionsExt}, - fs::{create_dir, remove_file, set_permissions, Permissions} -}; - -#[allow(unused_imports)] -use cfg_if::cfg_if; -use indexmap::IndexMap; - +mod build_support; -#[cfg(feature = "dwarfs")] -const DWARFS_VERSION: &str = "0.12.4"; -#[cfg(feature = "squashfs")] -const SQUASHFS_TOOLS_VERSION: &str = "4.6.1"; -#[cfg(feature = "squashfs")] -const SQUASHFUSE_VERSION: &str = "0.6.1"; +use std::{env, path::PathBuf}; +use build_support::{ + asset_urls, cache_generation_path, download_atomic, prepare_assets_with, target_spec, + BuildFeatures, +}; fn main() { - let arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); - - let project = env::var("CARGO_MANIFEST_DIR").unwrap(); - let project_path = Path::new(&project); + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=build_support.rs"); + println!("cargo:rerun-if-env-changed=TARGET"); + println!("cargo:rerun-if-env-changed=URUNTIME_CURL"); - let assets_path = project_path.join(format!("assets-{arch}")); - let assets_path_link = project_path.join("assets"); - - let assets = IndexMap::from([ - #[cfg(feature = "squashfs")] - ("squashfuse", format!("https://github.com/VHSgunzo/squashfuse-static/releases/download/v{SQUASHFUSE_VERSION}/squashfuse-musl-mimalloc-{arch}")), - #[cfg(feature = "squashfs")] - ("unsquashfs", format!("https://github.com/VHSgunzo/squashfs-tools-static/releases/download/v{SQUASHFS_TOOLS_VERSION}/unsquashfs-{arch}")), - #[cfg(all(not(feature = "lite"), feature = "squashfs"))] - ("mksquashfs", format!("https://github.com/VHSgunzo/squashfs-tools-static/releases/download/v{SQUASHFS_TOOLS_VERSION}/mksquashfs-{arch}")), - #[cfg(feature = "dwarfs")] - { - cfg_if! { - if #[cfg(feature = "lite")] { - ("dwarfs-fuse-extract-upx", format!("https://github.com/mhx/dwarfs/releases/download/v{0}/dwarfs-fuse-extract-{0}-Linux-{arch}", DWARFS_VERSION)) - } else { - ("dwarfs-universal-upx", format!("https://github.com/mhx/dwarfs/releases/download/v{0}/dwarfs-universal-{0}-Linux-{arch}", DWARFS_VERSION)) - } - } - }, - ]); - - if !assets_path.exists() { - create_dir(&assets_path).unwrap() + if let Err(err) = run() { + panic!("uruntime helper preparation failed: {err}"); } +} - let _ = remove_file(&assets_path_link); - symlink(&assets_path, &assets_path_link).unwrap(); - - for asset in assets.keys() { - #[allow(unused_mut)] - let mut asset = *asset; - #[allow(unused_mut)] - let mut asset_path = assets_path.join(asset); - #[allow(unused_mut)] - let mut asset_url = assets.get(asset).unwrap().clone(); - - #[cfg(feature = "upx")] - if !asset.ends_with("-upx") { - asset_path = assets_path.join(format!("{asset}-upx")); - asset_url = format!("{}-upx", asset_url) - } - - if !asset_path.exists() { - let output = Command::new("curl").args([ - "--insecure", - "-L", &asset_url, - "-o", asset_path.to_str().unwrap() - ]).output().unwrap_or_else(|err| panic!("Failed to execute curl: {err}: {asset}")); - - if !output.status.success() { - eprintln!("Failed to get asset: {}", String::from_utf8_lossy(&output.stderr)); - exit(1) - } - - set_permissions(&asset_path, Permissions::from_mode(0o755)) - .unwrap_or_else(|err| panic!("Unable to set permissions: {err}: {asset}")); - } - - #[cfg(not(feature = "upx"))] - { - if asset.ends_with("-upx") { - asset = asset.strip_suffix("-upx").unwrap(); - let asset_noupx_path = assets_path.join(asset); - if !asset_noupx_path.exists() { - let output = Command::new("upx").args([ - "-d", - asset_path.to_str().unwrap(), "-o", - asset_noupx_path.to_str().unwrap() - ]).output().unwrap_or_else(|err| panic!("Failed to execute upx: {err}")); - - if !output.status.success() { - eprintln!("Failed to decompress upx asset: {asset}: {}", String::from_utf8_lossy(&output.stderr)); - exit(1) - } - } - asset_path = asset_noupx_path - } - - let asset_zstd_path = assets_path.join(format!("{asset}-zst")); - if !asset_zstd_path.exists() { - let asset_data = std::fs::read(asset_path).unwrap(); - let asset_zstd_data = zstd::stream::encode_all(&asset_data[..], 22).unwrap(); - std::fs::write(asset_zstd_path, asset_zstd_data).unwrap(); - } - } - } +fn run() -> Result<(), String> { + build_support::validate_configuration()?; + let target_name = env::var("TARGET").map_err(|err| format!("TARGET is not set: {err}"))?; + let target = target_spec(&target_name)?; + let project = PathBuf::from( + env::var_os("CARGO_MANIFEST_DIR") + .ok_or_else(|| "CARGO_MANIFEST_DIR is not set".to_string())?, + ); + let out_dir = + PathBuf::from(env::var_os("OUT_DIR").ok_or_else(|| "OUT_DIR is not set".to_string())?); + let features = BuildFeatures { + squashfs: cfg!(feature = "squashfs"), + dwarfs: cfg!(feature = "dwarfs"), + lite: cfg!(feature = "lite"), + }; + let assets = asset_urls(target.release_arch, features); + let cache = project.join(cache_generation_path(target, features)); + let generated = out_dir.join("uruntime-helper-assets"); + let curl = env::var_os("URUNTIME_CURL").unwrap_or_else(|| "curl".into()); + + prepare_assets_with( + &cache, + &generated, + target, + &assets, + &|asset, destination| download_atomic(&curl, &asset.url, destination), + )?; + + let generated = generated + .canonicalize() + .map_err(|err| format!("failed to canonicalize {}: {err}", generated.display()))?; + println!( + "cargo:rustc-env=URUNTIME_HELPER_DIR={}", + generated.display() + ); + Ok(()) } diff --git a/build_support.rs b/build_support.rs new file mode 100644 index 0000000..d2c9b56 --- /dev/null +++ b/build_support.rs @@ -0,0 +1,1125 @@ +use std::{ + ffi::{OsStr, OsString}, + fmt, + fs::{self, File, OpenOptions}, + io::{Read, Write}, + path::{Path, PathBuf}, + process::{Command, Stdio}, + thread, + time::{Duration, Instant}, +}; + +use fs2::FileExt; +use sha2::{Digest, Sha256}; +use tempfile::{Builder as TempBuilder, NamedTempFile}; +use xxhash_rust::xxh64::xxh64; + +pub const DWARFS_VERSION: &str = "0.15.7"; + +pub const SQUASHFS_TOOLS_VERSION: &str = "4.7.5.r2"; +pub const SQUASHFUSE_VERSION: &str = "0.6.3.r2"; +pub const ZIG_VERSION: &str = "0.16.0"; +pub const ZIG_INDEX_URL: &str = "https://ziglang.org/download/index.json"; +pub const ZIG_DOWNLOAD_BASE: &str = "https://ziglang.org/download"; +pub const ZIG_DOWNLOAD_MAX: usize = 64 * 1024 * 1024; +pub const ZIG_PLATFORMS: [&str; 5] = [ + "aarch64-linux", + "loongarch64-linux", + "powerpc64le-linux", + "riscv64-linux", + "x86_64-linux", +]; +pub const CHECKSUM_MANIFEST: &str = include_str!("checksums.txt"); + +// Largest v0.15.7 wrapper: 2,965,645 bytes; largest r2 direct helper: +// 1,264,416 bytes. Eight MiB leaves >2.8x margin while bounding downloads/cache reads. +pub const MAX_DOWNLOAD_SIZE: usize = 8 * 1024 * 1024; +// Largest extracted v0.15.7 helper: 8,766,832 bytes. Sixteen MiB leaves >1.9x margin. +pub const MAX_HELPER_SIZE: usize = 16 * 1024 * 1024; +const LOCK_TIMEOUT: Duration = Duration::from_secs(120); +const LOCK_RETRY: Duration = Duration::from_millis(25); +const SQUEEZE_TRAILER_LEN: usize = 32; +const SQUEEZE_MAGIC: &[u8; 8] = b"SQUEEZE!"; + +pub fn validate_configuration() -> Result<(), String> { + for (name, version) in [ + ("DwarFS", DWARFS_VERSION), + ("squashfs-tools", SQUASHFS_TOOLS_VERSION), + ("squashfuse", SQUASHFUSE_VERSION), + ("Zig", ZIG_VERSION), + ] { + if version.is_empty() { + return Err(format!("{name} version must not be empty")); + } + } + if !ZIG_INDEX_URL.starts_with("https://ziglang.org/") + || !ZIG_DOWNLOAD_BASE.starts_with("https://ziglang.org/") + { + return Err("Zig index URL must use HTTPS on ziglang.org".into()); + } + if ZIG_DOWNLOAD_MAX < MAX_DOWNLOAD_SIZE || ZIG_PLATFORMS.is_empty() { + return Err("invalid Zig download limits or platform inventory".into()); + } + if !CHECKSUM_MANIFEST.contains("[helpers]") || !CHECKSUM_MANIFEST.contains("[zig]") { + return Err("checksums.txt must contain helpers and Zig sections".into()); + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ElfEndian { + Little, + Big, +} + +impl fmt::Display for ElfEndian { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Little => f.write_str("little-endian"), + Self::Big => f.write_str("big-endian"), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct TargetSpec { + pub release_arch: &'static str, + pub endian: ElfEndian, + pub elf_machine: u16, +} + +pub fn target_spec(target: &str) -> Result { + let spec = match target { + "x86_64-unknown-linux-musl" => TargetSpec { + release_arch: "x86_64", + endian: ElfEndian::Little, + elf_machine: 62, + }, + "aarch64-unknown-linux-musl" => TargetSpec { + release_arch: "aarch64", + endian: ElfEndian::Little, + elf_machine: 183, + }, + "riscv64gc-unknown-linux-musl" => TargetSpec { + release_arch: "riscv64", + endian: ElfEndian::Little, + elf_machine: 243, + }, + "loongarch64-unknown-linux-musl" => TargetSpec { + release_arch: "loongarch64", + endian: ElfEndian::Little, + elf_machine: 258, + }, + "powerpc64-unknown-linux-musl" => TargetSpec { + release_arch: "ppc64", + endian: ElfEndian::Big, + elf_machine: 21, + }, + "powerpc64le-unknown-linux-musl" => TargetSpec { + release_arch: "ppc64le", + endian: ElfEndian::Little, + elf_machine: 21, + }, + _ => { + return Err(format!( + "unsupported TARGET `{target}`; supported targets: \ + x86_64-unknown-linux-musl, aarch64-unknown-linux-musl, \ + riscv64gc-unknown-linux-musl, loongarch64-unknown-linux-musl, \ + powerpc64-unknown-linux-musl, powerpc64le-unknown-linux-musl" + )); + } + }; + Ok(spec) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct BuildFeatures { + pub squashfs: bool, + pub dwarfs: bool, + pub lite: bool, +} + +impl BuildFeatures { + pub fn cache_key(self) -> String { + format!( + "squashfs-{}_dwarfs-{}_lite-{}", + u8::from(self.squashfs), + u8::from(self.dwarfs), + u8::from(self.lite) + ) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum AssetKind { + Direct, + DwarfsWrapper, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ElfType { + /// squashfs-tools-static/squashfuse-static r2 files are static PIE (ET_DYN). + StaticPie, + /// DwarFS v0.15.7 wrapper payloads are static ET_EXEC files. + StaticExec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Asset { + pub name: &'static str, + pub url: String, + pub kind: AssetKind, + pub source_sha256: String, + pub payload_sha256: String, + pub elf_type: ElfType, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DigestRecord { + pub arch: String, + pub name: String, + pub source_sha256: String, + pub payload_sha256: String, +} + +fn valid_sha256(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +pub fn parse_digest_manifest(manifest: &str) -> Result, String> { + let mut records = Vec::new(); + let mut section = "[helpers]"; + for (index, line) in manifest.lines().enumerate() { + let line_number = index + 1; + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + if line.starts_with('[') && line.ends_with(']') { + section = line; + continue; + } + if section != "[helpers]" { + continue; + } + let fields: Vec<&str> = line.split('\t').collect(); + if fields.len() != 4 { + return Err(format!( + "checksum manifest line {line_number} has {} fields, expected 4 tab-separated fields", + fields.len() + )); + } + if !valid_sha256(fields[2]) || !valid_sha256(fields[3]) { + return Err(format!( + "checksum manifest line {line_number} contains an invalid SHA-256" + )); + } + records.push(DigestRecord { + arch: fields[0].to_string(), + name: fields[1].to_string(), + source_sha256: fields[2].to_ascii_lowercase(), + payload_sha256: fields[3].to_ascii_lowercase(), + }); + } + records.sort_by(|left, right| (&left.arch, &left.name).cmp(&(&right.arch, &right.name))); + for pair in records.windows(2) { + if pair[0].arch == pair[1].arch && pair[0].name == pair[1].name { + return Err(format!( + "duplicate checksum manifest entry for {}/{}", + pair[0].arch, pair[0].name + )); + } + } + Ok(records) +} + +pub fn digest_records() -> Result, String> { + parse_digest_manifest(CHECKSUM_MANIFEST) +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AssetSource { + pub target: TargetSpec, + pub name: &'static str, + pub url: String, + pub kind: AssetKind, + pub elf_type: ElfType, +} + +pub fn all_asset_sources() -> Vec { + const TARGETS: [&str; 6] = [ + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-musl", + "riscv64gc-unknown-linux-musl", + "loongarch64-unknown-linux-musl", + "powerpc64-unknown-linux-musl", + "powerpc64le-unknown-linux-musl", + ]; + let mut sources = Vec::with_capacity(30); + for target_name in TARGETS { + let target = target_spec(target_name).expect("static target table must be valid"); + let arch = target.release_arch; + sources.extend([ + AssetSource { + target, + name: "squashfuse", + url: format!("https://github.com/VHSgunzo/squashfuse-static/releases/download/v{SQUASHFUSE_VERSION}/squashfuse-musl-mimalloc-{arch}"), + kind: AssetKind::Direct, + elf_type: ElfType::StaticPie, + }, + AssetSource { + target, + name: "unsquashfs", + url: format!("https://github.com/VHSgunzo/squashfs-tools-static/releases/download/v{SQUASHFS_TOOLS_VERSION}/unsquashfs-{arch}"), + kind: AssetKind::Direct, + elf_type: ElfType::StaticPie, + }, + AssetSource { + target, + name: "mksquashfs", + url: format!("https://github.com/VHSgunzo/squashfs-tools-static/releases/download/v{SQUASHFS_TOOLS_VERSION}/mksquashfs-{arch}"), + kind: AssetKind::Direct, + elf_type: ElfType::StaticPie, + }, + AssetSource { + target, + name: "dwarfs-universal", + url: format!("https://github.com/mhx/dwarfs/releases/download/v{DWARFS_VERSION}/dwarfs-universal-{DWARFS_VERSION}-Linux-{arch}"), + kind: AssetKind::DwarfsWrapper, + elf_type: ElfType::StaticExec, + }, + AssetSource { + target, + name: "dwarfs-fuse-extract", + url: format!("https://github.com/mhx/dwarfs/releases/download/v{DWARFS_VERSION}/dwarfs-fuse-extract-{DWARFS_VERSION}-Linux-{arch}"), + kind: AssetKind::DwarfsWrapper, + elf_type: ElfType::StaticExec, + }, + ]); + } + sources.sort_by(|left, right| { + (left.target.release_arch, left.name).cmp(&(right.target.release_arch, right.name)) + }); + sources +} + +fn asset(source: &AssetSource) -> Asset { + let release_arch = source.target.release_arch; + let name = source.name; + let records = digest_records().unwrap_or_else(|error| panic!("invalid checksums.txt: {error}")); + let digest = records + .iter() + .find(|record| record.arch == release_arch && record.name == name) + .unwrap_or_else(|| panic!("missing digest manifest entry for {release_arch}/{name}")); + Asset { + name, + url: source.url.clone(), + kind: source.kind, + source_sha256: digest.source_sha256.clone(), + payload_sha256: digest.payload_sha256.clone(), + elf_type: source.elf_type, + } +} + +pub fn asset_urls(release_arch: &str, features: BuildFeatures) -> Vec { + let sources = all_asset_sources(); + let find = |name| { + sources + .iter() + .find(|source| source.target.release_arch == release_arch && source.name == name) + .unwrap_or_else(|| { + panic!("missing helper source inventory entry for {release_arch}/{name}") + }) + }; + let mut names = Vec::new(); + if features.squashfs { + names.extend(["squashfuse", "unsquashfs"]); + if !features.lite { + names.push("mksquashfs"); + } + } + if features.dwarfs { + names.push(if features.lite { + "dwarfs-fuse-extract" + } else { + "dwarfs-universal" + }); + } + names.into_iter().map(|name| asset(find(name))).collect() +} + +pub fn cache_version(squashfuse: &str, squashfs_tools: &str, dwarfs: &str) -> String { + format!("squashfuse-{squashfuse}_squashfs-tools-{squashfs_tools}_dwarfs-{dwarfs}") +} + +pub fn current_cache_version() -> String { + cache_version(SQUASHFUSE_VERSION, SQUASHFS_TOOLS_VERSION, DWARFS_VERSION) +} + +pub fn cache_relative_path(target: TargetSpec) -> PathBuf { + PathBuf::from(format!("assets-{}", target.release_arch)).join(current_cache_version()) +} + +pub fn cache_generation_path(target: TargetSpec, features: BuildFeatures) -> PathBuf { + cache_relative_path(target).join(features.cache_key()) +} + +fn checked_range( + offset: u64, + size: u64, + length: usize, + what: &str, +) -> Result, String> { + let offset = + usize::try_from(offset).map_err(|_| format!("{what} offset does not fit usize"))?; + let size = usize::try_from(size).map_err(|_| format!("{what} size does not fit usize"))?; + let end = offset + .checked_add(size) + .ok_or_else(|| format!("{what} range overflow"))?; + if end > length { + return Err(format!( + "{what} range {offset}..{end} exceeds file size {length}" + )); + } + Ok(offset..end) +} + +fn read_u16(data: &[u8], offset: usize, endian: ElfEndian) -> Result { + let bytes: [u8; 2] = data + .get(offset..offset + 2) + .ok_or_else(|| "truncated ELF field".to_string())? + .try_into() + .map_err(|_| "invalid ELF field".to_string())?; + Ok(match endian { + ElfEndian::Little => u16::from_le_bytes(bytes), + ElfEndian::Big => u16::from_be_bytes(bytes), + }) +} + +fn read_u32(data: &[u8], offset: usize, endian: ElfEndian) -> Result { + let bytes: [u8; 4] = data + .get(offset..offset + 4) + .ok_or_else(|| "truncated ELF field".to_string())? + .try_into() + .map_err(|_| "invalid ELF field".to_string())?; + Ok(match endian { + ElfEndian::Little => u32::from_le_bytes(bytes), + ElfEndian::Big => u32::from_be_bytes(bytes), + }) +} + +fn read_u64(data: &[u8], offset: usize, endian: ElfEndian) -> Result { + let bytes: [u8; 8] = data + .get(offset..offset + 8) + .ok_or_else(|| "truncated ELF field".to_string())? + .try_into() + .map_err(|_| "invalid ELF field".to_string())?; + Ok(match endian { + ElfEndian::Little => u64::from_le_bytes(bytes), + ElfEndian::Big => u64::from_be_bytes(bytes), + }) +} + +pub fn validate_elf(payload: &[u8], target: TargetSpec, elf_type: ElfType) -> Result<(), String> { + if payload.len() < 64 { + return Err("truncated ELF64 header".to_string()); + } + if &payload[..4] != b"\x7fELF" { + return Err("payload has invalid ELF magic".to_string()); + } + if payload[4] != 2 { + return Err(format!( + "unsupported ELF class {} (expected ELF64)", + payload[4] + )); + } + let expected_data = match target.endian { + ElfEndian::Little => 1, + ElfEndian::Big => 2, + }; + if payload[5] != expected_data { + return Err(format!( + "ELF endian {} does not match expected {}", + payload[5], target.endian + )); + } + if payload[6] != 1 { + return Err(format!( + "unsupported ELF identification version {}", + payload[6] + )); + } + let actual_type = read_u16(payload, 16, target.endian)?; + let expected_type = match elf_type { + ElfType::StaticPie => 3, + ElfType::StaticExec => 2, + }; + if actual_type != expected_type { + return Err(format!( + "ELF type {actual_type} does not match expected {expected_type:?} ({elf_type:?})" + )); + } + let machine = read_u16(payload, 18, target.endian)?; + if machine != target.elf_machine { + return Err(format!( + "ELF machine {machine} does not match expected {} for {}", + target.elf_machine, target.release_arch + )); + } + if read_u32(payload, 20, target.endian)? != 1 { + return Err("unsupported ELF header version".to_string()); + } + if read_u16(payload, 52, target.endian)? != 64 { + return Err("invalid ELF64 header size".to_string()); + } + let phoff = read_u64(payload, 32, target.endian)?; + let phentsize = usize::from(read_u16(payload, 54, target.endian)?); + let phnum = usize::from(read_u16(payload, 56, target.endian)?); + if phentsize != 56 || phnum == 0 { + return Err(format!( + "invalid ELF program header table dimensions: entry={phentsize}, count={phnum}" + )); + } + let table_size = phentsize + .checked_mul(phnum) + .ok_or_else(|| "ELF program header table size overflow".to_string())?; + let table = checked_range( + phoff, + table_size as u64, + payload.len(), + "ELF program header table", + )?; + + let mut executable_load = false; + for index in 0..phnum { + let offset = table.start + index * phentsize; + let p_type = read_u32(payload, offset, target.endian)?; + let flags = read_u32(payload, offset + 4, target.endian)?; + let file_offset = read_u64(payload, offset + 8, target.endian)?; + let file_size = read_u64(payload, offset + 32, target.endian)?; + let memory_size = read_u64(payload, offset + 40, target.endian)?; + if p_type == 1 && file_size > memory_size { + return Err(format!( + "PT_LOAD program header {index} has p_filesz > p_memsz" + )); + } + let range = checked_range( + file_offset, + file_size, + payload.len(), + &format!("program header {index}"), + )?; + match p_type { + 1 if flags & 1 != 0 => executable_load = true, + 3 => return Err("ELF contains forbidden PT_INTERP".to_string()), + 2 => { + if range.len() % 16 != 0 { + return Err( + "PT_DYNAMIC size is not a multiple of ELF64 dynamic entry size".to_string(), + ); + } + let mut terminated = false; + for dynamic_offset in (range.start..range.end).step_by(16) { + let tag = read_u64(payload, dynamic_offset, target.endian)?; + if tag == 0 { + terminated = true; + break; + } + if tag == 1 { + return Err("ELF contains forbidden DT_NEEDED".to_string()); + } + } + if !terminated { + return Err("unterminated PT_DYNAMIC table".to_string()); + } + } + _ => {} + } + } + if !executable_load { + return Err("ELF has no executable PT_LOAD segment".to_string()); + } + Ok(()) +} + +pub fn extract_dwarfs_wrapper( + wrapper: &[u8], + target: TargetSpec, + max_output_size: usize, +) -> Result, String> { + let trailer_offset = wrapper + .len() + .checked_sub(SQUEEZE_TRAILER_LEN) + .ok_or_else(|| "truncated DwarFS wrapper trailer".to_string())?; + let trailer = &wrapper[trailer_offset..]; + if &trailer[..8] != SQUEEZE_MAGIC { + return Err("invalid DwarFS wrapper magic".to_string()); + } + let field = |offset: usize| -> Result { + let bytes: [u8; 8] = trailer + .get(offset..offset + 8) + .ok_or_else(|| "truncated DwarFS wrapper trailer field".to_string())? + .try_into() + .map_err(|_| "invalid DwarFS wrapper trailer field".to_string())?; + Ok(u64::from_le_bytes(bytes)) + }; + let declared_size_u64 = field(8)?; + let compressed_size_u64 = field(16)?; + let expected_xxh64 = field(24)?; + let declared_size = usize::try_from(declared_size_u64).map_err(|_| { + format!("declared uncompressed size {declared_size_u64} does not fit usize") + })?; + if declared_size > max_output_size { + return Err(format!( + "declared uncompressed size {declared_size} exceeds limit {max_output_size}" + )); + } + let compressed_size = usize::try_from(compressed_size_u64).map_err(|_| { + format!("declared compressed size {compressed_size_u64} does not fit usize") + })?; + if compressed_size > MAX_DOWNLOAD_SIZE { + return Err(format!( + "declared compressed size {compressed_size} exceeds limit {MAX_DOWNLOAD_SIZE}" + )); + } + let payload_offset = trailer_offset.checked_sub(compressed_size).ok_or_else(|| { + format!("declared compressed size {compressed_size} exceeds wrapper payload area {trailer_offset}") + })?; + let compressed_end = payload_offset + .checked_add(compressed_size) + .ok_or_else(|| "compressed payload range overflow".to_string())?; + if compressed_end != trailer_offset { + return Err("compressed payload does not end at DwarFS trailer".to_string()); + } + + let decoder = zstd::stream::read::Decoder::new(&wrapper[payload_offset..compressed_end]) + .map_err(|err| format!("failed to initialize DwarFS Zstd decoder: {err}"))?; + let read_limit = declared_size + .checked_add(1) + .ok_or_else(|| "DwarFS declared output size overflow".to_string())?; + let mut payload = Vec::new(); + payload.try_reserve_exact(declared_size).map_err(|err| { + format!("failed to allocate {declared_size} bytes for DwarFS payload: {err}") + })?; + decoder + .take(read_limit as u64) + .read_to_end(&mut payload) + .map_err(|err| format!("failed to decode DwarFS Zstd payload: {err}"))?; + if payload.len() != declared_size { + return Err(format!( + "DwarFS uncompressed size mismatch: expected {declared_size}, got {}", + payload.len() + )); + } + let actual_xxh64 = xxh64(&payload, 0); + if actual_xxh64 != expected_xxh64 { + return Err(format!( + "DwarFS XXH64 mismatch: expected {expected_xxh64:016x}, got {actual_xxh64:016x}" + )); + } + validate_elf(&payload, target, ElfType::StaticExec)?; + Ok(payload) +} + +fn read_bounded(path: &Path, max_size: usize) -> Result, String> { + let file = + File::open(path).map_err(|err| format!("failed to open {}: {err}", path.display()))?; + let metadata = file + .metadata() + .map_err(|err| format!("failed to inspect {}: {err}", path.display()))?; + if !metadata.is_file() { + return Err(format!("{} is not a regular file", path.display())); + } + if metadata.len() > max_size as u64 { + return Err(format!( + "{} size {} exceeds limit {max_size}", + path.display(), + metadata.len() + )); + } + let capacity = usize::try_from(metadata.len()) + .map_err(|_| format!("{} size does not fit usize", path.display()))?; + let mut data = Vec::new(); + data.try_reserve_exact(capacity).map_err(|err| { + format!( + "failed to allocate {capacity} bytes for {}: {err}", + path.display() + ) + })?; + file.take((max_size as u64).saturating_add(1)) + .read_to_end(&mut data) + .map_err(|err| format!("failed to read {}: {err}", path.display()))?; + if data.len() > max_size { + return Err(format!("{} exceeds limit {max_size}", path.display())); + } + Ok(data) +} + +pub fn sha256_hex(data: &[u8]) -> String { + format!("{:x}", Sha256::digest(data)) +} + +pub fn sha256_file(path: &Path, max_size: usize) -> Result { + Ok(sha256_hex(&read_bounded(path, max_size)?)) +} + +fn verify_digest(path: &Path, expected: &str, max_size: usize) -> Result<(), String> { + let actual = sha256_file(path, max_size)?; + if actual != expected { + return Err(format!( + "SHA-256 mismatch for {}: expected {expected}, got {actual}", + path.display() + )); + } + Ok(()) +} + +pub fn atomic_write(destination: &Path, data: &[u8]) -> Result<(), String> { + let parent = destination + .parent() + .ok_or_else(|| format!("destination has no parent: {}", destination.display()))?; + fs::create_dir_all(parent) + .map_err(|err| format!("failed to create {}: {err}", parent.display()))?; + let mut temporary = NamedTempFile::new_in(parent).map_err(|err| { + format!( + "failed to create secure temporary file in {}: {err}", + parent.display() + ) + })?; + temporary.write_all(data).map_err(|err| { + format!( + "failed to write temporary file for {}: {err}", + destination.display() + ) + })?; + temporary.flush().map_err(|err| { + format!( + "failed to flush temporary file for {}: {err}", + destination.display() + ) + })?; + temporary.as_file().sync_all().map_err(|err| { + format!( + "failed to sync temporary file for {}: {err}", + destination.display() + ) + })?; + temporary.persist(destination).map_err(|err| { + format!( + "failed to atomically publish {}: {}", + destination.display(), + err.error + ) + })?; + File::open(parent) + .and_then(|directory| directory.sync_all()) + .map_err(|err| format!("failed to sync directory {}: {err}", parent.display()))?; + Ok(()) +} + +pub fn curl_args(url: &str, max_size: usize) -> Vec { + [ + "--fail".to_string(), + "--location".to_string(), + "--retry".to_string(), + "3".to_string(), + "--max-filesize".to_string(), + max_size.to_string(), + url.to_string(), + ] + .into_iter() + .collect() +} + +pub fn download_atomic>( + curl: S, + url: &str, + destination: &Path, +) -> Result<(), String> { + let parent = destination + .parent() + .ok_or_else(|| format!("destination has no parent: {}", destination.display()))?; + fs::create_dir_all(parent) + .map_err(|err| format!("failed to create {}: {err}", parent.display()))?; + let temporary = NamedTempFile::new_in(parent).map_err(|err| { + format!( + "failed to create secure download file in {}: {err}", + parent.display() + ) + })?; + let stdout_file = temporary + .reopen() + .map_err(|err| format!("failed to reopen secure download file: {err}"))?; + let output = Command::new(curl.as_ref()) + .args(curl_args(url, MAX_DOWNLOAD_SIZE)) + .stdout(Stdio::from(stdout_file)) + .stderr(Stdio::piped()) + .output() + .map_err(|err| format!("failed to execute {:?} for {url}: {err}", curl.as_ref()))?; + if !output.status.success() { + return Err(format!( + "failed to download {url}: {}", + String::from_utf8_lossy(&output.stderr).trim() + )); + } + let size = temporary + .as_file() + .metadata() + .map_err(|err| format!("failed to inspect download for {url}: {err}"))? + .len(); + if size > MAX_DOWNLOAD_SIZE as u64 { + return Err(format!( + "download from {url} is {size} bytes, exceeding limit {MAX_DOWNLOAD_SIZE}" + )); + } + temporary.persist(destination).map_err(|err| { + format!( + "failed to atomically publish {}: {}", + destination.display(), + err.error + ) + })?; + Ok(()) +} + +struct CacheLock { + file: File, +} + +impl CacheLock { + fn acquire(cache: &Path) -> Result { + let parent = cache + .parent() + .ok_or_else(|| format!("cache has no parent: {}", cache.display()))?; + fs::create_dir_all(parent) + .map_err(|err| format!("failed to create {}: {err}", parent.display()))?; + let name = cache + .file_name() + .ok_or_else(|| format!("cache has no file name: {}", cache.display()))?; + let mut lock_name = OsString::from("."); + lock_name.push(name); + lock_name.push(".lock"); + let path = parent.join(lock_name); + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&path) + .map_err(|err| format!("failed to open cache lock {}: {err}", path.display()))?; + let started = Instant::now(); + loop { + match file.try_lock_exclusive() { + Ok(()) => return Ok(Self { file }), + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => { + if started.elapsed() >= LOCK_TIMEOUT { + return Err(format!( + "timed out waiting for cache lock {}", + path.display() + )); + } + thread::sleep(LOCK_RETRY); + } + Err(err) => return Err(format!("failed to lock {}: {err}", path.display())), + } + } + } +} + +impl Drop for CacheLock { + fn drop(&mut self) { + let _ = self.file.unlock(); + } +} + +#[allow(dead_code)] +pub fn with_cache_lock(path: &Path, operation: F) -> Result +where + F: FnOnce() -> Result, +{ + let _lock = CacheLock::acquire(path)?; + operation() +} + +fn source_name(asset: &Asset) -> OsString { + let mut name = OsString::from(".source-"); + name.push(asset.name); + name +} + +pub fn source_cache_name(name: &str, kind: AssetKind) -> String { + match kind { + AssetKind::Direct => name.to_string(), + AssetKind::DwarfsWrapper => format!("{name}-wrapper"), + } +} + +fn source_cache_component(name: &str) -> Result { + match name { + "squashfuse" => Ok(format!("squashfuse-{SQUASHFUSE_VERSION}")), + "unsquashfs" | "mksquashfs" => Ok(format!("squashfs-tools-{SQUASHFS_TOOLS_VERSION}")), + "dwarfs-universal" | "dwarfs-fuse-extract" => Ok(format!("dwarfs-{DWARFS_VERSION}")), + _ => Err(format!("unknown helper source `{name}`")), + } +} + +#[allow(dead_code)] +pub fn source_cache_relative_path( + target: TargetSpec, + name: &str, + kind: AssetKind, +) -> Result { + Ok(PathBuf::from(format!("assets-{}", target.release_arch)) + .join(source_cache_component(name)?) + .join(source_cache_name(name, kind))) +} + +fn source_cache_path(cache: &Path, target: TargetSpec, asset: &Asset) -> Result { + let architecture_dir = format!("assets-{}", target.release_arch); + let root = cache + .ancestors() + .find(|path| path.file_name() == Some(OsStr::new(&architecture_dir))) + .or_else(|| cache.parent()) + .ok_or_else(|| format!("cache has no parent directory: {}", cache.display()))?; + Ok(root + .join(source_cache_component(asset.name)?) + .join(source_cache_name(asset.name, asset.kind))) +} + +fn generation_valid(cache: &Path, target: TargetSpec, assets: &[Asset]) -> Result<(), String> { + for asset in assets { + verify_digest( + &cache.join(source_name(asset)), + &asset.source_sha256, + MAX_DOWNLOAD_SIZE, + )?; + let raw = read_bounded(&cache.join(asset.name), MAX_HELPER_SIZE)?; + if sha256_hex(&raw) != asset.payload_sha256 { + return Err(format!("cached {} payload SHA-256 mismatch", asset.name)); + } + validate_elf(&raw, target, asset.elf_type)?; + let compressed_path = cache.join(format!("{}-zst", asset.name)); + let compressed = read_bounded(&compressed_path, MAX_DOWNLOAD_SIZE)?; + let decoded = decode_zstd_bounded(&compressed, raw.len())?; + if decoded != raw { + return Err(format!( + "cached {}-zst does not decode to cached raw ELF", + asset.name + )); + } + } + Ok(()) +} + +fn decode_zstd_bounded(compressed: &[u8], declared_size: usize) -> Result, String> { + if declared_size > MAX_HELPER_SIZE { + return Err(format!( + "declared Zstd output {declared_size} exceeds {MAX_HELPER_SIZE}" + )); + } + let decoder = zstd::stream::read::Decoder::new(compressed) + .map_err(|err| format!("failed to initialize Zstd decoder: {err}"))?; + let limit = declared_size + .checked_add(1) + .ok_or_else(|| "Zstd output size overflow".to_string())?; + let mut decoded = Vec::new(); + decoded + .try_reserve_exact(declared_size) + .map_err(|err| format!("failed to allocate {declared_size} decoded bytes: {err}"))?; + decoder + .take(limit as u64) + .read_to_end(&mut decoded) + .map_err(|err| format!("failed to decode Zstd data: {err}"))?; + if decoded.len() != declared_size { + return Err(format!( + "Zstd output size mismatch: expected {declared_size}, got {}", + decoded.len() + )); + } + Ok(decoded) +} + +fn copy_bounded(source: &Path, destination: &Path, max_size: usize) -> Result<(), String> { + let data = read_bounded(source, max_size)?; + atomic_write(destination, &data) +} + +fn replace_directory(stage: PathBuf, destination: &Path) -> Result<(), String> { + let parent = destination + .parent() + .ok_or_else(|| format!("destination has no parent: {}", destination.display()))?; + let backup = TempBuilder::new() + .prefix(".uruntime-backup-") + .tempdir_in(parent) + .map_err(|err| { + format!( + "failed to reserve backup path in {}: {err}", + parent.display() + ) + })?; + let backup_path = backup.path().to_path_buf(); + backup + .close() + .map_err(|err| format!("failed to release backup path: {err}"))?; + let had_destination = destination.exists(); + if had_destination { + fs::rename(destination, &backup_path) + .map_err(|err| format!("failed to move {} to backup: {err}", destination.display()))?; + } + if let Err(err) = fs::rename(&stage, destination) { + if had_destination { + if let Err(rollback_err) = fs::rename(&backup_path, destination) { + return Err(format!( + "failed to publish {}: {err}; rollback also failed: {rollback_err}; previous generation remains at {}", + destination.display(), + backup_path.display() + )); + } + } + return Err(format!( + "failed to publish {}: {err}", + destination.display() + )); + } + if had_destination { + fs::remove_dir_all(&backup_path).map_err(|err| { + format!( + "failed to remove cache backup {}: {err}", + backup_path.display() + ) + })?; + } + Ok(()) +} + +pub fn stage_output(source: &Path, destination: &Path, files: &[&str]) -> Result<(), String> { + let parent = destination + .parent() + .ok_or_else(|| format!("output has no parent: {}", destination.display()))?; + fs::create_dir_all(parent) + .map_err(|err| format!("failed to create {}: {err}", parent.display()))?; + let stage = TempBuilder::new() + .prefix(".uruntime-output-") + .tempdir_in(parent) + .map_err(|err| { + format!( + "failed to create output stage in {}: {err}", + parent.display() + ) + })?; + for name in files { + let max = if name.ends_with("-zst") { + MAX_DOWNLOAD_SIZE + } else { + MAX_HELPER_SIZE + }; + copy_bounded(&source.join(name), &stage.path().join(name), max)?; + } + let stage_path = stage.keep(); + replace_directory(stage_path, destination) +} + +pub fn prepare_assets_with( + cache: &Path, + output: &Path, + target: TargetSpec, + assets: &[Asset], + downloader: &F, +) -> Result<(), String> +where + F: Fn(&Asset, &Path) -> Result<(), String>, +{ + let _lock = CacheLock::acquire(cache)?; + if generation_valid(cache, target, assets).is_err() { + let parent = cache + .parent() + .ok_or_else(|| format!("cache has no parent: {}", cache.display()))?; + fs::create_dir_all(parent) + .map_err(|err| format!("failed to create {}: {err}", parent.display()))?; + let stage = TempBuilder::new() + .prefix(".uruntime-cache-stage-") + .tempdir_in(parent) + .map_err(|err| { + format!( + "failed to create cache stage in {}: {err}", + parent.display() + ) + })?; + + for asset in assets { + let stage_source = stage.path().join(source_name(asset)); + let cached_source = cache.join(source_name(asset)); + if verify_digest(&cached_source, &asset.source_sha256, MAX_DOWNLOAD_SIZE).is_ok() { + copy_bounded(&cached_source, &stage_source, MAX_DOWNLOAD_SIZE)?; + } else { + let source_cache = source_cache_path(cache, target, asset)?; + { + let _source_lock = CacheLock::acquire(&source_cache)?; + if verify_digest(&source_cache, &asset.source_sha256, MAX_DOWNLOAD_SIZE) + .is_err() + { + downloader(asset, &source_cache)?; + verify_digest(&source_cache, &asset.source_sha256, MAX_DOWNLOAD_SIZE) + .map_err(|err| { + let _ = fs::remove_file(&source_cache); + format!("downloaded asset failed integrity validation: {err}") + })?; + } + copy_bounded(&source_cache, &stage_source, MAX_DOWNLOAD_SIZE)?; + } + } + + let source = read_bounded(&stage_source, MAX_DOWNLOAD_SIZE)?; + let payload = match asset.kind { + AssetKind::Direct => source, + AssetKind::DwarfsWrapper => { + extract_dwarfs_wrapper(&source, target, MAX_HELPER_SIZE)? + } + }; + let actual_payload = sha256_hex(&payload); + if actual_payload != asset.payload_sha256 { + return Err(format!( + "payload SHA-256 mismatch for {}: expected {}, got {actual_payload}", + asset.name, asset.payload_sha256 + )); + } + validate_elf(&payload, target, asset.elf_type)?; + atomic_write(&stage.path().join(asset.name), &payload)?; + let compressed = zstd::stream::encode_all(&payload[..], 22) + .map_err(|err| format!("failed to Zstd-compress {}: {err}", asset.name))?; + if compressed.len() > MAX_DOWNLOAD_SIZE { + return Err(format!( + "compressed {} size {} exceeds {MAX_DOWNLOAD_SIZE}", + asset.name, + compressed.len() + )); + } + let decoded = decode_zstd_bounded(&compressed, payload.len())?; + if decoded != payload { + return Err(format!( + "generated {}-zst failed byte-equality verification", + asset.name + )); + } + atomic_write( + &stage.path().join(format!("{}-zst", asset.name)), + &compressed, + )?; + } + generation_valid(stage.path(), target, assets)?; + let stage_path = stage.keep(); + replace_directory(stage_path, cache)?; + } + + let mut files = Vec::with_capacity(assets.len() * 2); + for asset in assets { + files.push(asset.name); + } + let compressed_names: Vec = assets + .iter() + .map(|asset| format!("{}-zst", asset.name)) + .collect(); + let mut all_names: Vec<&str> = files; + all_names.extend(compressed_names.iter().map(String::as_str)); + stage_output(cache, output, &all_names) +} diff --git a/checksums.txt b/checksums.txt new file mode 100644 index 0000000..f9ec352 --- /dev/null +++ b/checksums.txt @@ -0,0 +1,42 @@ +# uruntime checksum manifest v1 + +[helpers] +# arch name source_sha256 payload_sha256 +aarch64 dwarfs-fuse-extract edb698186db6b162c755770ffabfcc05482bb7105a716fd943548e64244b0fd2 cc3b3e342ca6537139d8c90d3c0a00236cf786bb61bdccf019e5ed0ca5917a7a +aarch64 dwarfs-universal 7e07a81edc6b029470512c6829f61e590734d4a67905675474afd6f273f48a9b 36716cce6082c3b03836a0a2aa0e28381e16f96417e77d4f860e1663e168abae +aarch64 mksquashfs d0f2dd468d6a6ca13eb252bbdce36cb5f9e5628f92630c7a1dc0031ef36c141e d0f2dd468d6a6ca13eb252bbdce36cb5f9e5628f92630c7a1dc0031ef36c141e +aarch64 squashfuse a9d5866b0f45894a8047ad776be844f7175416c956a79955a7a100a33d0cfd93 a9d5866b0f45894a8047ad776be844f7175416c956a79955a7a100a33d0cfd93 +aarch64 unsquashfs 106751249b612a1570e24131f5dacaaadb21d7c7b0d07e1cd0f7012417ae6fdb 106751249b612a1570e24131f5dacaaadb21d7c7b0d07e1cd0f7012417ae6fdb +loongarch64 dwarfs-fuse-extract 920b5cb8998daab2c4eb1bcc43d9f2f771831835e0574978a8aa90068f84f30c b982a1b23f221eca56ba12998659eb2829df34707eeeb2454290264917237990 +loongarch64 dwarfs-universal b9302afb42e4ea487d5d96cab7a2a004348a6425b835ad597509e04ad5712bca 984571a8af1b802d3e6419d722c0e69a0689ef5194c9fa290187412ab596bca1 +loongarch64 mksquashfs 853906a62679a04f5f202056a6aed9d5186bb7dbf1cef9b972261aa632089e57 853906a62679a04f5f202056a6aed9d5186bb7dbf1cef9b972261aa632089e57 +loongarch64 squashfuse 7bafe7870146edd461597523c42ccc9a295a7c465ffe884f9e53c08adfbbabd2 7bafe7870146edd461597523c42ccc9a295a7c465ffe884f9e53c08adfbbabd2 +loongarch64 unsquashfs 77693a7457f0bd18dad76b0d6c3ed9af4c838e1dc91917dba4e27359e82ce931 77693a7457f0bd18dad76b0d6c3ed9af4c838e1dc91917dba4e27359e82ce931 +ppc64 dwarfs-fuse-extract aa17302ece213eee867421373d602eb05a4c27934441904f5f54e8df79bd3480 ea796585a40f584cb6c8c2bbde1a615e78e44486d9710560cb8e139ad1ad6baf +ppc64 dwarfs-universal 60bc6a07df35d59e93eafe53a6d20c61fa45cdf38e22fb3806bea00f5de8626d 2fee1d16abf9bbfc3bc76c5ee6d74305d3b1e43a47d8d10a5be48a233ab86d0a +ppc64 mksquashfs ed2da819cabfa4540c61b68d8530ab4c7dfa658fca56a364c46adaaac04c9aff ed2da819cabfa4540c61b68d8530ab4c7dfa658fca56a364c46adaaac04c9aff +ppc64 squashfuse 531cf26c98e95bad6e5bffa745ba4eeff80aaee75372262f5861cfe2a09f6d3e 531cf26c98e95bad6e5bffa745ba4eeff80aaee75372262f5861cfe2a09f6d3e +ppc64 unsquashfs daa963f2aee3d2556775f55b7294f4d0a2145a0c3f074f264c2175565fe1106c daa963f2aee3d2556775f55b7294f4d0a2145a0c3f074f264c2175565fe1106c +ppc64le dwarfs-fuse-extract 4454698ed95aa337c72cc840d105eb425b21ad730c18d8faa43bdec94525f1fb 9a7e284d0ac11ef304cfbf97695e109215fed78742855b33f212622a0f30f03c +ppc64le dwarfs-universal 6c50a92a85f8edff672323ba3fa5e55309f6961c39eac6caddc18c5a083d76c2 c0365bd9e494ad20d5061eb489d80786ea50f6f53f6d15edd1538abdfba253a4 +ppc64le mksquashfs fdda3f0eb4e602c3d0e5ef3615c3ad3bf9d0ab857cc66047ba9694f49600b440 fdda3f0eb4e602c3d0e5ef3615c3ad3bf9d0ab857cc66047ba9694f49600b440 +ppc64le squashfuse fc00008826dcea3d047da9cf6fb32737ef1983697a5c7d5e576d77d79d7f9ae1 fc00008826dcea3d047da9cf6fb32737ef1983697a5c7d5e576d77d79d7f9ae1 +ppc64le unsquashfs 1b1c05b5956b4d673878cf664f25a163ef0eb3632f710fd4de5fba9cd3e0f81b 1b1c05b5956b4d673878cf664f25a163ef0eb3632f710fd4de5fba9cd3e0f81b +riscv64 dwarfs-fuse-extract 3339208ba117601584dba7d8c2ea270bf7a36112e05a902c9a6cf76eed70920a b9cf2d5f03c5ec4b802ee7e01f2164586ed79ccbeeb79e2c0937291cea4850b8 +riscv64 dwarfs-universal c8048684d8f0e182ac630d4724d8dbbc869e375833b5ba16159f668f11251742 09b816ecb382521540b41bc42344a23db07c6b3fb970cc04a609ab29f8eb3167 +riscv64 mksquashfs 7491ad853c77ca05ebab111d8bbfc0061cd68e1295a412d99c05bc2ac09bb26e 7491ad853c77ca05ebab111d8bbfc0061cd68e1295a412d99c05bc2ac09bb26e +riscv64 squashfuse 6e509181b35e88548a910e8b821d594e514b5cbb07594047eb1877f2eb0625b1 6e509181b35e88548a910e8b821d594e514b5cbb07594047eb1877f2eb0625b1 +riscv64 unsquashfs b6334b378b719a2e766c0e1ca45a48a71e88c14ce33c1be57b600bfca601ead9 b6334b378b719a2e766c0e1ca45a48a71e88c14ce33c1be57b600bfca601ead9 +x86_64 dwarfs-fuse-extract 72c106576087e8049f4c669cbf2cec4765201982a035faf65a6ef6dfef3819db 2c230635d98cec69aa66a4c5fad6166bc70f21f50864a8e31b1a6fa73bec36d2 +x86_64 dwarfs-universal baa03026e7d2c195fdb78bf261cd7d20f620b20685f8a3e26162ba9d652b0d78 f4cb43ae5f3858305ed1f3a6874bb17e5302de2a85e912500b053235886a4525 +x86_64 mksquashfs 083df1d372a93955f64ba5f12f1f9232fa70c2fc34e616dceba12f40a2320e2c 083df1d372a93955f64ba5f12f1f9232fa70c2fc34e616dceba12f40a2320e2c +x86_64 squashfuse 8a55524d05b5ba6513c303d664f675ccc4e0eb795d2bb3f5bed6833379c307dd 8a55524d05b5ba6513c303d664f675ccc4e0eb795d2bb3f5bed6833379c307dd +x86_64 unsquashfs dba5813d159032005c49862a23224c9b1b4e97d579a0b1afe6e30e1bc0384520 dba5813d159032005c49862a23224c9b1b4e97d579a0b1afe6e30e1bc0384520 + +[zig] +# version platform url sha256 +0.16.0 aarch64-linux https://ziglang.org/download/0.16.0/zig-aarch64-linux-0.16.0.tar.xz ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17 +0.16.0 loongarch64-linux https://ziglang.org/download/0.16.0/zig-loongarch64-linux-0.16.0.tar.xz 2503be8ecc5965f1f7962471d267d9f83fcb3cc2f7ff78ac34093b9722bbea93 +0.16.0 powerpc64le-linux https://ziglang.org/download/0.16.0/zig-powerpc64le-linux-0.16.0.tar.xz 18800b45c08bf40b335ca5ab79aea70aca287ca969036e938155772becaeebeb +0.16.0 riscv64-linux https://ziglang.org/download/0.16.0/zig-riscv64-linux-0.16.0.tar.xz bc069b0f2f568f54bafbdfc1d65b12fd386ed6a652044a37aee6a4f72f14076e +0.16.0 x86_64-linux https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz 70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00 diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..5d56faf --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "nightly" diff --git a/scripts/ci_artifacts.py b/scripts/ci_artifacts.py new file mode 100644 index 0000000..c1a2593 --- /dev/null +++ b/scripts/ci_artifacts.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 +"""Validate and aggregate uruntime CI artifacts without trusting filenames alone.""" + +from __future__ import annotations + +import argparse +import json +import os +import resource +import signal +import stat +import struct +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class ArchSpec: + machine: int + endian: str + qemu: str | None + + +ARCHES = { + "x86_64": ArchSpec(62, "little", None), + "aarch64": ArchSpec(183, "little", "qemu-aarch64-static"), + "riscv64": ArchSpec(243, "little", "qemu-riscv64-static"), + "loongarch64": ArchSpec(258, "little", "qemu-loongarch64-static"), + "ppc64": ArchSpec(21, "big", "qemu-ppc64-static"), + "ppc64le": ArchSpec(21, "little", "qemu-ppc64le-static"), +} + +VARIANTS = ( + "runimage", + "runimage-squashfs", + "runimage-dwarfs", + "appimage", + "appimage-lite", + "appimage-squashfs", + "appimage-squashfs-lite", + "appimage-dwarfs", + "appimage-dwarfs-lite", +) + +REQUIRED_SECTIONS = frozenset( + {".envs", ".upd_info", ".sig_key", ".sha256_sig", ".digest_md5"} +) +PT_INTERP = 3 +PT_DYNAMIC = 2 +DT_NULL = 0 +DT_NEEDED = 1 +PN_XNUM = 0xFFFF +MAX_ARTIFACT_SIZE = 64 * 1024 * 1024 +MAX_PROGRAM_HEADERS = 128 +MAX_SECTION_HEADERS = 1024 +MAX_SECTION_NAME_TABLE = 1024 * 1024 +MAX_SMOKE_OUTPUT = 64 * 1024 + + +def expected_artifact_names(arch: str) -> list[str]: + if arch not in ARCHES: + raise ValueError(f"unsupported architecture: {arch}") + return [f"uruntime-{variant}-{arch}" for variant in VARIANTS] + + +def expected_all_artifact_names() -> list[str]: + return [name for arch in ARCHES for name in expected_artifact_names(arch)] + + +def _range(offset: int, size: int, length: int, description: str) -> range: + if offset < 0 or size < 0 or offset > length or size > length - offset: + raise ValueError( + f"{description} range {offset}..{offset + size} exceeds file size {length}" + ) + return range(offset, offset + size) + + +def _read_regular(path: Path, limit: int = MAX_ARTIFACT_SIZE) -> bytes: + if path.is_symlink(): + raise ValueError(f"{path}: symlink is not allowed") + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as error: + raise ValueError(f"cannot open {path}: {error}") from error + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + raise ValueError(f"{path}: not a regular file") + if metadata.st_size > limit: + raise ValueError(f"{path}: file size {metadata.st_size} exceeds size limit {limit}") + with os.fdopen(descriptor, "rb", closefd=False) as source: + data = source.read(limit + 1) + if len(data) > limit: + raise ValueError(f"{path}: file exceeds size limit {limit}") + return data + finally: + os.close(descriptor) + + +def _section_names(table: bytes) -> dict[int, str]: + names = {} + offset = 0 + while offset < len(table): + end = table.find(b"\0", offset) + if end < 0: + raise ValueError("unterminated section name table") + try: + names[offset] = table[offset:end].decode("ascii") + except UnicodeDecodeError as error: + raise ValueError("non-ASCII section name") from error + offset = end + 1 + return names + + +def validate_elf(path: Path, arch: str, expected_magic: bytes) -> None: + data = _read_regular(path) + _validate_elf_bytes(data, path, arch, expected_magic) + + +def _validate_elf_bytes( + data: bytes, path: Path | str, arch: str, expected_magic: bytes +) -> None: + spec = ARCHES[arch] + if len(data) < 64 or data[:4] != b"\x7fELF": + raise ValueError(f"{path}: not an ELF64 file") + if data[4] != 2: + raise ValueError(f"{path}: expected ELF64 class, got {data[4]}") + expected_data = 1 if spec.endian == "little" else 2 + if data[5] != expected_data: + raise ValueError( + f"{path}: ELF endian byte {data[5]} does not match {spec.endian}" + ) + if data[6] != 1: + raise ValueError(f"{path}: unsupported ELF version {data[6]}") + if data[8:11] != expected_magic: + raise ValueError( + f"{path}: runtime magic {data[8:11].hex()} does not match {expected_magic.hex()}" + ) + + order = "<" if spec.endian == "little" else ">" + machine = struct.unpack_from(order + "H", data, 18)[0] + if machine != spec.machine: + raise ValueError( + f"{path}: ELF machine {machine} does not match {spec.machine} for {arch}" + ) + ehsize = struct.unpack_from(order + "H", data, 52)[0] + if ehsize != 64: + raise ValueError(f"{path}: invalid ELF64 header size {ehsize}") + + phoff = struct.unpack_from(order + "Q", data, 32)[0] + phentsize, phnum = struct.unpack_from(order + "HH", data, 54) + if phnum == PN_XNUM: + raise ValueError(f"{path}: extended program-header numbering is unsupported") + if phentsize != 56 or phnum == 0 or phnum > MAX_PROGRAM_HEADERS: + raise ValueError( + f"{path}: invalid program header dimensions ({phentsize}, {phnum})" + ) + ph_table = _range(phoff, phentsize * phnum, len(data), "program header table") + for index in range(phnum): + offset = ph_table.start + index * phentsize + p_type = struct.unpack_from(order + "I", data, offset)[0] + p_offset = struct.unpack_from(order + "Q", data, offset + 8)[0] + p_filesz = struct.unpack_from(order + "Q", data, offset + 32)[0] + segment = _range(p_offset, p_filesz, len(data), f"program header {index}") + if p_type == PT_INTERP: + raise ValueError(f"{path}: static contract violated by PT_INTERP") + if p_type == PT_DYNAMIC: + if len(segment) % 16: + raise ValueError(f"{path}: malformed PT_DYNAMIC table") + terminated = False + for dynamic_offset in range(segment.start, segment.stop, 16): + tag = struct.unpack_from(order + "Q", data, dynamic_offset)[0] + if tag == DT_NULL: + terminated = True + break + if tag == DT_NEEDED: + raise ValueError(f"{path}: static contract violated by DT_NEEDED") + if not terminated: + raise ValueError(f"{path}: unterminated PT_DYNAMIC table") + + shoff = struct.unpack_from(order + "Q", data, 40)[0] + shentsize, shnum, shstrndx = struct.unpack_from(order + "HHH", data, 58) + if ( + shentsize != 64 + or shnum == 0 + or shnum > MAX_SECTION_HEADERS + or shstrndx == 0xFFFF + or shstrndx >= shnum + ): + raise ValueError( + f"{path}: invalid or unsupported section header dimensions/index " + f"({shentsize}, {shnum}, {shstrndx})" + ) + sh_table = _range(shoff, shentsize * shnum, len(data), "section header table") + + def section_header(index: int) -> tuple[int, int, int, int]: + offset = sh_table.start + index * shentsize + name_offset, section_type = struct.unpack_from(order + "II", data, offset) + file_offset, size = struct.unpack_from(order + "QQ", data, offset + 24) + return name_offset, section_type, file_offset, size + + _, _, names_offset, names_size = section_header(shstrndx) + if names_size > MAX_SECTION_NAME_TABLE: + raise ValueError(f"{path}: section name table exceeds size limit") + names_range = _range(names_offset, names_size, len(data), "section string table") + names_table = data[names_range.start:names_range.stop] + section_names = _section_names(names_table) + present = set() + for index in range(1, shnum): + name_offset, section_type, file_offset, size = section_header(index) + if section_type != 8: # SHT_NOBITS has no file-backed range. + _range(file_offset, size, len(data), f"section {index}") + try: + present.add(section_names[name_offset]) + except KeyError as error: + raise ValueError(f"section name offset {name_offset} is not a string boundary") from error + missing = sorted(REQUIRED_SECTIONS - present) + if missing: + raise ValueError(f"{path}: missing required runtime section(s): {', '.join(missing)}") + + +def _directory_files(directory: Path) -> set[str]: + if directory.is_symlink(): + raise ValueError(f"artifact path must not be a symlink: {directory}") + if not directory.is_dir(): + raise ValueError(f"artifact path is not a directory: {directory}") + entries = list(directory.iterdir()) + invalid = sorted(entry.name for entry in entries if not entry.is_file() or entry.is_symlink()) + if invalid: + raise ValueError(f"non-regular artifact entries: {', '.join(invalid)}") + return {entry.name for entry in entries} + + +def _write_exclusive(path: Path, data: bytes, mode: int = 0o755) -> None: + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags, mode) + except OSError as error: + raise ValueError(f"cannot create exclusive output {path}: {error}") from error + try: + with os.fdopen(descriptor, "wb", closefd=False) as output: + output.write(data) + output.flush() + os.fsync(descriptor) + except OSError as error: + try: + path.unlink() + except OSError: + pass + raise ValueError(f"cannot stage {path}: {error}") from error + finally: + os.close(descriptor) + + +def _set_smoke_limits() -> None: + resource.setrlimit(resource.RLIMIT_FSIZE, (MAX_SMOKE_OUTPUT, MAX_SMOKE_OUTPUT)) + + +def _smoke_validated_bytes(arch: str, name: str, data: bytes) -> None: + if name not in expected_artifact_names(arch): + raise ValueError(f"unexpected smoke-test artifact name: {name}") + with tempfile.TemporaryDirectory(prefix="uruntime-smoke-") as temporary: + private_binary = Path(temporary) / name + _write_exclusive(private_binary, data, 0o700) + command = smoke_command(arch, private_binary) + try: + with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr: + result = subprocess.run( + command, + check=False, + stdout=stdout, + stderr=stderr, + timeout=30, + preexec_fn=_set_smoke_limits, + ) + stdout.seek(0) + stderr.seek(0) + stdout_bytes = stdout.read(MAX_SMOKE_OUTPUT + 1) + stderr_bytes = stderr.read(MAX_SMOKE_OUTPUT + 1) + except (OSError, subprocess.TimeoutExpired) as error: + raise ValueError(f"smoke test failed to execute {name}: {error}") from error + if ( + result.returncode in (-signal.SIGXFSZ, 128 + signal.SIGXFSZ) + or len(stdout_bytes) > MAX_SMOKE_OUTPUT + or len(stderr_bytes) > MAX_SMOKE_OUTPUT + ): + raise ValueError(f"smoke test output limit exceeded for {name}") + stderr_text = stderr_bytes.decode(errors="replace") + if result.returncode != 0: + raise ValueError( + f"smoke test failed for {name} ({result.returncode}): {stderr_text.strip()}" + ) + stdout_text = stdout_bytes.decode(errors="replace") + if not stdout_text.strip().startswith("v"): + raise ValueError(f"smoke test returned unexpected version for {name}") + + +def validate_arch(directory: Path, arch: str, smoke: bool = False) -> None: + if arch not in ARCHES: + raise ValueError(f"unsupported architecture: {arch}") + expected = set(expected_artifact_names(arch)) + actual = _directory_files(directory) + missing = sorted(expected - actual) + unexpected = sorted(actual - expected) + if missing or unexpected: + raise ValueError( + f"artifact manifest mismatch for {arch}; missing={missing}, unexpected={unexpected}" + ) + for name in sorted(expected): + path = directory / name + magic = b"AI\x02" if name.startswith("uruntime-appimage") else b"RI\x02" + data = _read_regular(path) + _validate_elf_bytes(data, path, arch, magic) + if smoke: + _smoke_validated_bytes(arch, name, data) + print(f"validated {len(expected)} {arch} artifacts") + + +def smoke_command(arch: str, binary: Path) -> list[str]: + if arch not in ARCHES: + raise ValueError(f"unsupported architecture: {arch}") + prefix = "appimage" if binary.name.startswith("uruntime-appimage") else "runtime" + command = [str(binary), f"--{prefix}-version"] + qemu = ARCHES[arch].qemu + if qemu: + command.insert(0, qemu) + return command + + +def aggregate_release(downloads: Path, output: Path) -> None: + expected_directories = {f"uruntime-{arch}" for arch in ARCHES} + if downloads.is_symlink(): + raise ValueError(f"download path must not be a symlink: {downloads}") + if not downloads.is_dir(): + raise ValueError(f"download path is not a directory: {downloads}") + actual_directories = {entry.name for entry in downloads.iterdir()} + if actual_directories != expected_directories: + raise ValueError( + "unexpected artifact directories; " + f"missing={sorted(expected_directories - actual_directories)}, " + f"unexpected={sorted(actual_directories - expected_directories)}" + ) + if output.is_symlink(): + raise ValueError(f"release output must not be a symlink: {output}") + if output.exists() and any(output.iterdir()): + raise ValueError(f"release output is not empty (stale files): {output}") + output.mkdir(parents=True, exist_ok=True) + + seen = set() + for arch in ARCHES: + source_directory = downloads / f"uruntime-{arch}" + expected_arch = set(expected_artifact_names(arch)) + actual_arch = _directory_files(source_directory) + if actual_arch != expected_arch: + raise ValueError( + f"artifact manifest mismatch for {arch}; " + f"missing={sorted(expected_arch - actual_arch)}, " + f"unexpected={sorted(actual_arch - expected_arch)}" + ) + for name in expected_artifact_names(arch): + if name in seen: + raise ValueError(f"duplicate release artifact: {name}") + source = source_directory / name + magic = b"AI\x02" if name.startswith("uruntime-appimage") else b"RI\x02" + data = _read_regular(source) + _validate_elf_bytes(data, source, arch, magic) + seen.add(name) + _write_exclusive(output / name, data) + + expected = set(expected_all_artifact_names()) + if seen != expected or _directory_files(output) != expected: + raise ValueError("release staging manifest does not contain exactly 54 artifacts") + print(f"staged {len(seen)} release artifacts in {output}") + + +def validate_release(release_json: Path) -> None: + try: + assets = json.loads(release_json.read_text()) + if not isinstance(assets, list): + raise TypeError("release assets payload must be a JSON list") + names = [asset["name"] for asset in assets] + except (OSError, json.JSONDecodeError, KeyError, TypeError) as error: + raise ValueError(f"invalid release metadata: {error}") from error + expected = set(expected_all_artifact_names()) + if len(names) != len(set(names)) or set(names) != expected: + raise ValueError( + "published release manifest mismatch; " + f"missing={sorted(expected - set(names))}, unexpected={sorted(set(names) - expected)}" + ) + print("validated exact 54-asset published release manifest") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + arch_parser = subparsers.add_parser("validate-arch") + arch_parser.add_argument("arch", choices=ARCHES) + arch_parser.add_argument("directory", type=Path) + arch_parser.add_argument("--smoke", action="store_true") + aggregate_parser = subparsers.add_parser("aggregate-release") + aggregate_parser.add_argument("downloads", type=Path) + aggregate_parser.add_argument("output", type=Path) + release_parser = subparsers.add_parser("validate-release") + release_parser.add_argument("metadata", type=Path) + args = parser.parse_args(argv) + try: + if args.command == "validate-arch": + validate_arch(args.directory, args.arch, args.smoke) + elif args.command == "aggregate-release": + aggregate_release(args.downloads, args.output) + else: + validate_release(args.metadata) + except ValueError as error: + parser.exit(1, f"error: {error}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/zig-linker.sh b/scripts/zig-linker.sh new file mode 100755 index 0000000..446d400 --- /dev/null +++ b/scripts/zig-linker.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${URUNTIME_RUST_TARGET:?URUNTIME_RUST_TARGET is required}" +: "${URUNTIME_ZIG_TARGET:?URUNTIME_ZIG_TARGET is required}" +: "${URUNTIME_RUST_SYSROOT:?URUNTIME_RUST_SYSROOT is required}" + +case "$URUNTIME_RUST_TARGET" in + x86_64-unknown-linux-musl) expected=x86_64-linux-musl ;; + aarch64-unknown-linux-musl) expected=aarch64-linux-musl ;; + riscv64gc-unknown-linux-musl) expected=riscv64-linux-musl ;; + loongarch64-unknown-linux-musl) expected=loongarch64-linux-musl ;; + powerpc64-unknown-linux-musl) expected=powerpc64-linux-musl ;; + powerpc64le-unknown-linux-musl) expected=powerpc64le-linux-musl ;; + *) printf 'zig-linker: unsupported Rust target: %s\n' "$URUNTIME_RUST_TARGET" >&2; exit 2 ;; +esac + +if [[ "$URUNTIME_ZIG_TARGET" != "$expected" ]]; then + printf 'zig-linker: target mismatch: Rust %s maps to Zig %s, not %s\n' \ + "$URUNTIME_RUST_TARGET" "$expected" "$URUNTIME_ZIG_TARGET" >&2 + exit 2 +fi + +rust_crt_dir="$URUNTIME_RUST_SYSROOT/lib/rustlib/$URUNTIME_RUST_TARGET/lib/self-contained" +rust_target_lib_dir="$URUNTIME_RUST_SYSROOT/lib/rustlib/$URUNTIME_RUST_TARGET/lib" +args=() +skip_target_value=false +pending_library_dir=false +for arg in "$@"; do + if $skip_target_value; then + skip_target_value=false + continue + fi + if $pending_library_dir; then + pending_library_dir=false + if [[ "$arg" == "$rust_target_lib_dir" && ! -d "$arg" ]]; then + continue + fi + args+=("-L" "$arg") + continue + fi + case "$arg" in + --target|-target) + skip_target_value=true + ;; + --target=*|-target=*) + ;; + -L) + pending_library_dir=true + ;; + -L"$rust_target_lib_dir") + [[ -d "$rust_target_lib_dir" ]] && args+=("$arg") + ;; + -Wl,--fix-cortex-a53-843419) + ;; + -nostartfiles|-lc) + ;; + "$rust_crt_dir/crt1.o"|"$rust_crt_dir/Scrt1.o"|"$rust_crt_dir/rcrt1.o"|\ + "$rust_crt_dir/crti.o"|"$rust_crt_dir/crtn.o"|"$rust_crt_dir/crtbegin.o"|\ + "$rust_crt_dir/crtbeginS.o"|"$rust_crt_dir/crtend.o"|"$rust_crt_dir/crtendS.o") + ;; + *) + args+=("$arg") + ;; + esac +done + +if $pending_library_dir; then + args+=("-L") +fi + +zig=${URUNTIME_ZIG:-zig} +exec "$zig" cc -target "$URUNTIME_ZIG_TARGET" "${args[@]}" diff --git a/src/elf_layout.rs b/src/elf_layout.rs new file mode 100644 index 0000000..7f32ab1 --- /dev/null +++ b/src/elf_layout.rs @@ -0,0 +1,244 @@ +use std::io::{Error, ErrorKind::InvalidData, Read, Result}; + +#[derive(Clone, Copy)] +enum Endian { + Little, + Big, +} + +impl Endian { + fn u16(self, bytes: &[u8]) -> Result { + let raw = bytes + .try_into() + .map_err(|_| invalid_data("truncated 16-bit ELF field"))?; + Ok(match self { + Self::Little => u16::from_le_bytes(raw), + Self::Big => u16::from_be_bytes(raw), + }) + } + + fn u32(self, bytes: &[u8]) -> Result { + let raw = bytes + .try_into() + .map_err(|_| invalid_data("truncated 32-bit ELF field"))?; + Ok(match self { + Self::Little => u32::from_le_bytes(raw), + Self::Big => u32::from_be_bytes(raw), + }) + } + + fn u64(self, bytes: &[u8]) -> Result { + let raw = bytes + .try_into() + .map_err(|_| invalid_data("truncated 64-bit ELF field"))?; + Ok(match self { + Self::Little => u64::from_le_bytes(raw), + Self::Big => u64::from_be_bytes(raw), + }) + } +} + +const ELF64_HEADER_SIZE: u64 = 64; +const ELF64_PROGRAM_HEADER_SIZE: u64 = 56; +const ELF64_SECTION_HEADER_SIZE: u64 = 64; +// Bound attacker-controlled seeks and allocations while leaving ample room for the +// compressed helpers and custom sections in supported uruntime binaries. +const MAX_ELF_TABLE_BYTES: u64 = 8 * 1024 * 1024; +const MAX_ELF_PREFIX_BYTES: u64 = 128 * 1024 * 1024; +const SHT_NOBITS: u32 = 8; +const PN_XNUM: u16 = u16::MAX; +const SHN_XINDEX: u16 = u16::MAX; + +fn invalid_data(message: impl Into) -> Error { + Error::new(InvalidData, message.into()) +} + +fn checked_end(offset: u64, size: u64, file_len: u64, label: &str) -> Result { + let end = offset + .checked_add(size) + .ok_or_else(|| invalid_data(format!("{label} range overflows u64")))?; + if end > file_len { + return Err(invalid_data(format!( + "{label} range {offset}..{end} exceeds file length {file_len}" + ))); + } + Ok(end) +} + +fn append_exact( + reader: &mut R, + bytes: &mut Vec, + additional: u64, + label: &str, +) -> Result<()> { + let additional = usize::try_from(additional) + .map_err(|_| invalid_data(format!("{label} size does not fit usize")))?; + bytes + .try_reserve_exact(additional) + .map_err(|error| invalid_data(format!("cannot allocate {label}: {error}")))?; + let read = reader + .take(additional as u64) + .read_to_end(bytes) + .map_err(|error| invalid_data(format!("cannot read {label}: {error}")))?; + if read != additional { + return Err(invalid_data(format!( + "cannot read {label}: expected {additional} bytes, read {read}" + ))); + } + Ok(()) +} + +pub(crate) struct ElfPrefix { + pub(crate) boundary: u64, + pub(crate) bytes: Vec, +} + +pub(crate) fn read_elf_prefix(reader: &mut R, file_len: u64) -> Result { + if file_len < ELF64_HEADER_SIZE { + return Err(invalid_data("file is shorter than an ELF64 header")); + } + + let mut header = [0u8; ELF64_HEADER_SIZE as usize]; + reader + .read_exact(&mut header) + .map_err(|error| invalid_data(format!("cannot read ELF64 header: {error}")))?; + if &header[..4] != b"\x7fELF" { + return Err(invalid_data("invalid ELF magic")); + } + if header[4] != 2 { + return Err(invalid_data(format!( + "unsupported ELF EI_CLASS value {} (expected ELF64)", + header[4] + ))); + } + if header[6] != 1 { + return Err(invalid_data("invalid ELF identification version")); + } + + let endian = match header[5] { + 1 => Endian::Little, + 2 => Endian::Big, + value => { + return Err(invalid_data(format!( + "unsupported ELF EI_DATA value {value}" + ))) + } + }; + let phoff = endian.u64(&header[32..40])?; + let shoff = endian.u64(&header[40..48])?; + let ehsize = u64::from(endian.u16(&header[52..54])?); + let phentsize = u64::from(endian.u16(&header[54..56])?); + let phnum = endian.u16(&header[56..58])?; + let shentsize = u64::from(endian.u16(&header[58..60])?); + let shnum = endian.u16(&header[60..62])?; + let shstrndx = endian.u16(&header[62..64])?; + + if ehsize != ELF64_HEADER_SIZE { + return Err(invalid_data(format!( + "invalid ELF64 e_ehsize {ehsize} (expected {ELF64_HEADER_SIZE})" + ))); + } + if phnum == PN_XNUM { + return Err(invalid_data( + "extended ELF program-header numbering is unsupported", + )); + } + if phnum == 0 { + if phoff != 0 || !matches!(phentsize, 0 | ELF64_PROGRAM_HEADER_SIZE) { + return Err(invalid_data("inconsistent absent ELF program-header table")); + } + } else if phoff == 0 || phentsize != ELF64_PROGRAM_HEADER_SIZE { + return Err(invalid_data( + "invalid ELF64 program-header table offset or e_phentsize", + )); + } + if shstrndx == SHN_XINDEX { + return Err(invalid_data( + "extended ELF section-name indexing is unsupported", + )); + } + if shstrndx != 0 && shstrndx >= shnum { + return Err(invalid_data(format!( + "ELF section-name table index {shstrndx} is outside {shnum} section headers" + ))); + } + if (shoff == 0) != (shnum == 0) { + return Err(invalid_data( + "extended or inconsistent ELF section-header numbering is unsupported", + )); + } + if shnum == 0 { + if !matches!(shentsize, 0 | ELF64_SECTION_HEADER_SIZE) { + return Err(invalid_data("inconsistent absent ELF section-header table")); + } + } else if shentsize != ELF64_SECTION_HEADER_SIZE { + return Err(invalid_data(format!( + "invalid ELF64 e_shentsize {shentsize} (expected {ELF64_SECTION_HEADER_SIZE})" + ))); + } + + let ph_table_size = phentsize + .checked_mul(u64::from(phnum)) + .ok_or_else(|| invalid_data("program-header table size overflows u64"))?; + let sh_table_size = shentsize + .checked_mul(u64::from(shnum)) + .ok_or_else(|| invalid_data("section-header table size overflows u64"))?; + if ph_table_size > MAX_ELF_TABLE_BYTES || sh_table_size > MAX_ELF_TABLE_BYTES { + return Err(invalid_data("ELF metadata table exceeds safety limit")); + } + + let ph_table_end = checked_end(phoff, ph_table_size, file_len, "program-header table")?; + let sh_table_end = checked_end(shoff, sh_table_size, file_len, "section-header table")?; + if ph_table_end > MAX_ELF_PREFIX_BYTES || sh_table_end > MAX_ELF_PREFIX_BYTES { + return Err(invalid_data("ELF metadata table lies beyond safety limit")); + } + let metadata_end = ehsize.max(ph_table_end).max(sh_table_end); + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(metadata_end as usize) + .map_err(|error| invalid_data(format!("cannot allocate ELF metadata: {error}")))?; + bytes.extend_from_slice(&header); + append_exact( + reader, + &mut bytes, + metadata_end - ELF64_HEADER_SIZE, + "ELF metadata", + )?; + + let mut boundary = metadata_end; + + if ph_table_size != 0 { + let table = &bytes[phoff as usize..ph_table_end as usize]; + for entry in table.chunks_exact(phentsize as usize) { + let offset = endian.u64(&entry[8..16])?; + let size = endian.u64(&entry[32..40])?; + if size != 0 { + boundary = boundary.max(checked_end(offset, size, file_len, "program segment")?); + } + } + } + + if sh_table_size != 0 { + let table = &bytes[shoff as usize..sh_table_end as usize]; + for entry in table.chunks_exact(shentsize as usize) { + let section_type = endian.u32(&entry[4..8])?; + let size = endian.u64(&entry[32..40])?; + if section_type == SHT_NOBITS || size == 0 { + continue; + } + let offset = endian.u64(&entry[24..32])?; + boundary = boundary.max(checked_end(offset, size, file_len, "section")?); + } + } + + if boundary > MAX_ELF_PREFIX_BYTES { + return Err(invalid_data(format!( + "ELF file-backed prefix size {boundary} exceeds safety limit {MAX_ELF_PREFIX_BYTES}" + ))); + } + append_exact(reader, &mut bytes, boundary - metadata_end, "ELF prefix")?; + Ok(ElfPrefix { boundary, bytes }) +} + +#[cfg(test)] +pub(crate) mod tests; diff --git a/src/elf_layout/tests.rs b/src/elf_layout/tests.rs new file mode 100644 index 0000000..abe3c10 --- /dev/null +++ b/src/elf_layout/tests.rs @@ -0,0 +1,368 @@ +use super::read_elf_prefix; +use std::io::{Cursor, Read, Result}; + +struct CountingCursor { + inner: Cursor>, + bytes_read: usize, +} + +impl CountingCursor { + fn new(bytes: Vec) -> Self { + Self { + inner: Cursor::new(bytes), + bytes_read: 0, + } + } +} + +impl Read for CountingCursor { + fn read(&mut self, buffer: &mut [u8]) -> Result { + let count = self.inner.read(buffer)?; + self.bytes_read += count; + Ok(count) + } +} + +#[derive(Clone, Copy)] +pub(crate) enum Endian { + Little, + Big, +} + +impl Endian { + fn ei_data(self) -> u8 { + match self { + Self::Little => 1, + Self::Big => 2, + } + } +} + +fn put_u16(bytes: &mut [u8], offset: usize, value: u16, endian: Endian) { + let raw = match endian { + Endian::Little => value.to_le_bytes(), + Endian::Big => value.to_be_bytes(), + }; + bytes[offset..offset + 2].copy_from_slice(&raw); +} + +fn put_u32(bytes: &mut [u8], offset: usize, value: u32, endian: Endian) { + let raw = match endian { + Endian::Little => value.to_le_bytes(), + Endian::Big => value.to_be_bytes(), + }; + bytes[offset..offset + 4].copy_from_slice(&raw); +} + +fn put_u64(bytes: &mut [u8], offset: usize, value: u64, endian: Endian) { + let raw = match endian { + Endian::Little => value.to_le_bytes(), + Endian::Big => value.to_be_bytes(), + }; + bytes[offset..offset + 8].copy_from_slice(&raw); +} + +pub(crate) fn fixture(endian: Endian) -> Vec { + const PHOFF: usize = 64; + const SHOFF: usize = 0x180; + const SECTION_DATA_END: usize = 0x320; + const SEGMENT_END: usize = 0x380; + + let mut bytes = vec![0; SEGMENT_END]; + bytes[0..4].copy_from_slice(b"\x7fELF"); + bytes[4] = 2; + bytes[5] = endian.ei_data(); + bytes[6] = 1; + put_u16(&mut bytes, 16, 2, endian); + put_u16(&mut bytes, 18, 21, endian); + put_u32(&mut bytes, 20, 1, endian); + put_u64(&mut bytes, 32, PHOFF as u64, endian); + put_u64(&mut bytes, 40, SHOFF as u64, endian); + put_u16(&mut bytes, 52, 64, endian); + put_u16(&mut bytes, 54, 56, endian); + put_u16(&mut bytes, 56, 1, endian); + put_u16(&mut bytes, 58, 64, endian); + put_u16(&mut bytes, 60, 4, endian); + put_u16(&mut bytes, 62, 2, endian); + + put_u32(&mut bytes, PHOFF, 1, endian); + put_u64(&mut bytes, PHOFF + 8, 0x340, endian); + put_u64(&mut bytes, PHOFF + 32, 0x40, endian); + + let first = SHOFF + 64; + put_u32(&mut bytes, first, 17, endian); + put_u32(&mut bytes, first + 4, 1, endian); + put_u64(&mut bytes, first + 24, 0x300, endian); + put_u64( + &mut bytes, + first + 32, + (SECTION_DATA_END - 0x300) as u64, + endian, + ); + + let second = SHOFF + 128; + put_u32(&mut bytes, second, 1, endian); + put_u32(&mut bytes, second + 4, 3, endian); + put_u64(&mut bytes, second + 24, 0x120, endian); + put_u64(&mut bytes, second + 32, 26, endian); + bytes[0x120..0x120 + 26].copy_from_slice(b"\0.shstrtab\0.envs\0.payload\0"); + + let third = SHOFF + 192; + put_u32(&mut bytes, third, 11, endian); + put_u32(&mut bytes, third + 4, 1, endian); + put_u64(&mut bytes, third + 24, 0x150, endian); + put_u64(&mut bytes, third + 32, 16, endian); + bytes[0x150..0x158].copy_from_slice(b"VALUE=1\0"); + + bytes.extend_from_slice(b"hsqs"); + bytes.extend_from_slice(&[0x5a; 4096]); + bytes +} + +#[test] +fn reads_each_prefix_byte_only_once() { + let bytes = fixture(Endian::Little); + let mut reader = CountingCursor::new(bytes.clone()); + + let prefix = read_elf_prefix(&mut reader, bytes.len() as u64).unwrap(); + + assert_eq!(prefix.boundary, 0x380); + assert_eq!(reader.bytes_read, prefix.boundary as usize); +} + +#[test] +fn powerpc64le_fixture_uses_little_endian_boundary() { + let bytes = fixture(Endian::Little); + let mut cursor = Cursor::new(&bytes); + + let prefix = read_elf_prefix(&mut cursor, bytes.len() as u64).unwrap(); + + assert_eq!(prefix.boundary, 0x380); + assert_eq!(prefix.bytes.len(), 0x380); + assert_eq!(&bytes[prefix.boundary as usize..][..4], b"hsqs"); +} + +#[test] +fn powerpc64_big_endian_fixture_has_same_boundary() { + let bytes = fixture(Endian::Big); + let mut cursor = Cursor::new(&bytes); + + let prefix = read_elf_prefix(&mut cursor, bytes.len() as u64).unwrap(); + + assert_eq!(prefix.boundary, 0x380); + assert_eq!(prefix.bytes.len(), 0x380); +} + +#[test] +fn unknown_elf_data_encoding_is_invalid_data() { + let mut bytes = fixture(Endian::Little); + bytes[5] = 0; + let mut cursor = Cursor::new(&bytes); + + let error = match read_elf_prefix(&mut cursor, bytes.len() as u64) { + Ok(_) => panic!("invalid EI_DATA was accepted"), + Err(error) => error, + }; + + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); +} + +fn assert_invalid(bytes: Vec) { + let file_len = bytes.len() as u64; + assert_invalid_with_len(bytes, file_len); +} + +fn assert_invalid_with_len(bytes: Vec, file_len: u64) { + let mut cursor = Cursor::new(&bytes); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + read_elf_prefix(&mut cursor, file_len) + })); + let error = match result { + Ok(Err(error)) => error, + Ok(Ok(_)) => panic!("malformed ELF was accepted"), + Err(_) => panic!("malformed ELF caused a panic"), + }; + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); +} + +#[test] +fn malformed_headers_fail_closed_as_invalid_data() { + assert_invalid(vec![0; 10]); + + let mut bad_magic = fixture(Endian::Little); + bad_magic[0] = 0; + assert_invalid(bad_magic); + + let mut bad_class = fixture(Endian::Little); + bad_class[4] = 1; + assert_invalid(bad_class); + + let mut bad_ehsize = fixture(Endian::Little); + put_u16(&mut bad_ehsize, 52, 63, Endian::Little); + assert_invalid(bad_ehsize); + + let mut bad_phentsize = fixture(Endian::Little); + put_u16(&mut bad_phentsize, 54, 55, Endian::Little); + assert_invalid(bad_phentsize); + + let mut bad_shentsize = fixture(Endian::Little); + put_u16(&mut bad_shentsize, 58, 63, Endian::Little); + assert_invalid(bad_shentsize); + + let mut bad_absent_phentsize = fixture(Endian::Little); + put_u64(&mut bad_absent_phentsize, 32, 0, Endian::Little); + put_u16(&mut bad_absent_phentsize, 56, 0, Endian::Little); + put_u16(&mut bad_absent_phentsize, 54, 55, Endian::Little); + assert_invalid(bad_absent_phentsize); + + let mut bad_absent_shentsize = fixture(Endian::Little); + put_u64(&mut bad_absent_shentsize, 40, 0, Endian::Little); + put_u16(&mut bad_absent_shentsize, 60, 0, Endian::Little); + put_u16(&mut bad_absent_shentsize, 62, 0, Endian::Little); + put_u16(&mut bad_absent_shentsize, 58, 63, Endian::Little); + assert_invalid(bad_absent_shentsize); + + let mut extended_phnum = fixture(Endian::Little); + put_u16(&mut extended_phnum, 56, u16::MAX, Endian::Little); + assert_invalid(extended_phnum); + + let mut extended_shnum = fixture(Endian::Little); + put_u16(&mut extended_shnum, 60, 0, Endian::Little); + assert_invalid(extended_shnum); + + let mut extended_shstrndx = fixture(Endian::Little); + put_u16(&mut extended_shstrndx, 62, u16::MAX, Endian::Little); + assert_invalid(extended_shstrndx); + + let mut out_of_range_shstrndx = fixture(Endian::Little); + put_u16(&mut out_of_range_shstrndx, 62, 4, Endian::Little); + assert_invalid(out_of_range_shstrndx); +} + +#[test] +fn invalid_ranges_and_overflow_fail_without_panicking() { + let mut table_past_eof = fixture(Endian::Little); + let table_past_eof_offset = table_past_eof.len() as u64 - 32; + put_u64( + &mut table_past_eof, + 40, + table_past_eof_offset, + Endian::Little, + ); + assert_invalid(table_past_eof); + + let mut segment_past_eof = fixture(Endian::Little); + let segment_past_eof_size = segment_past_eof.len() as u64; + put_u64(&mut segment_past_eof, 64 + 8, 0x300, Endian::Little); + put_u64( + &mut segment_past_eof, + 64 + 32, + segment_past_eof_size, + Endian::Little, + ); + assert_invalid(segment_past_eof); + + let mut segment_overflow = fixture(Endian::Little); + put_u64(&mut segment_overflow, 64 + 8, u64::MAX, Endian::Little); + put_u64(&mut segment_overflow, 64 + 32, 2, Endian::Little); + assert_invalid_with_len(segment_overflow, u64::MAX); + + let mut section_overflow = fixture(Endian::Little); + let section = 0x180 + 64; + put_u64( + &mut section_overflow, + section + 24, + u64::MAX, + Endian::Little, + ); + put_u64(&mut section_overflow, section + 32, 2, Endian::Little); + assert_invalid_with_len(section_overflow, u64::MAX); + + let mut remote_table = fixture(Endian::Little); + put_u64(&mut remote_table, 40, 0x1_0000_0000, Endian::Little); + assert_invalid_with_len(remote_table, u64::MAX); + + let mut table_overflow = fixture(Endian::Little); + put_u64(&mut table_overflow, 40, u64::MAX - 32, Endian::Little); + assert_invalid_with_len(table_overflow, u64::MAX); + + let mut oversized_prefix = fixture(Endian::Little); + put_u64( + &mut oversized_prefix, + 64 + 8, + super::MAX_ELF_PREFIX_BYTES, + Endian::Little, + ); + put_u64(&mut oversized_prefix, 64 + 32, 1, Endian::Little); + assert_invalid_with_len(oversized_prefix, u64::MAX); +} + +#[test] +fn header_and_table_ranges_can_each_define_boundary() { + let mut header_only = fixture(Endian::Little); + put_u64(&mut header_only, 32, 0, Endian::Little); + put_u16(&mut header_only, 54, 0, Endian::Little); + put_u16(&mut header_only, 56, 0, Endian::Little); + put_u64(&mut header_only, 40, 0, Endian::Little); + put_u16(&mut header_only, 58, 0, Endian::Little); + put_u16(&mut header_only, 60, 0, Endian::Little); + put_u16(&mut header_only, 62, 0, Endian::Little); + let mut cursor = Cursor::new(&header_only); + let prefix = read_elf_prefix(&mut cursor, header_only.len() as u64).unwrap(); + assert_eq!(prefix.boundary, 64); + + let mut program_table = header_only.clone(); + put_u64(&mut program_table, 32, 0x80, Endian::Little); + put_u16(&mut program_table, 54, 56, Endian::Little); + put_u16(&mut program_table, 56, 1, Endian::Little); + let mut cursor = Cursor::new(&program_table); + let prefix = read_elf_prefix(&mut cursor, program_table.len() as u64).unwrap(); + assert_eq!(prefix.boundary, 0xb8); + + let mut section_table = header_only; + put_u64(&mut section_table, 40, 0x180, Endian::Little); + put_u16(&mut section_table, 58, 64, Endian::Little); + put_u16(&mut section_table, 60, 1, Endian::Little); + let mut cursor = Cursor::new(§ion_table); + let prefix = read_elf_prefix(&mut cursor, section_table.len() as u64).unwrap(); + assert_eq!(prefix.boundary, 0x1c0); +} + +#[test] +fn elf_without_section_table_is_supported() { + let mut bytes = fixture(Endian::Big); + put_u64(&mut bytes, 40, 0, Endian::Big); + put_u16(&mut bytes, 58, 0, Endian::Big); + put_u16(&mut bytes, 60, 0, Endian::Big); + put_u16(&mut bytes, 62, 0, Endian::Big); + let mut cursor = Cursor::new(&bytes); + + let prefix = read_elf_prefix(&mut cursor, bytes.len() as u64).unwrap(); + + assert_eq!(prefix.boundary, 0x380); +} + +#[test] +fn unordered_section_headers_use_largest_file_end() { + let mut bytes = fixture(Endian::Little); + put_u64(&mut bytes, 64 + 32, 0, Endian::Little); + let mut cursor = Cursor::new(&bytes); + + let prefix = read_elf_prefix(&mut cursor, bytes.len() as u64).unwrap(); + + assert_eq!(prefix.boundary, 0x320); +} + +#[test] +fn nobits_section_does_not_occupy_file_bytes() { + let mut bytes = fixture(Endian::Little); + let section = 0x180 + 64; + put_u32(&mut bytes, section + 4, 8, Endian::Little); + put_u64(&mut bytes, section + 24, 0x10_0000, Endian::Little); + put_u64(&mut bytes, section + 32, 0x20_0000, Endian::Little); + let mut cursor = Cursor::new(&bytes); + + let prefix = read_elf_prefix(&mut cursor, bytes.len() as u64).unwrap(); + + assert_eq!(prefix.boundary, 0x380); +} diff --git a/src/main.rs b/src/main.rs index 0e47cd3..31d407b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,28 +1,61 @@ use std::{ - str, path::PathBuf, - thread::{sleep, spawn}, - process::{exit, Command}, env::{self, current_exe}, - time::{self, Duration, Instant}, + fs::{ + self, create_dir, create_dir_all, read_to_string, remove_dir, remove_dir_all, remove_file, + set_permissions, File, Metadata, Permissions, + }, hash::{DefaultHasher, Hash, Hasher}, - io::{Error, ErrorKind::{NotFound, InvalidData}, Read, Write, Result, Seek, SeekFrom}, - os::unix::{prelude::PermissionsExt, fs::{symlink, MetadataExt}, process::CommandExt}, - fs::{self, File, Permissions, create_dir, create_dir_all, remove_dir, remove_dir_all, remove_file, set_permissions, read_to_string}, + io::{ + Error, + ErrorKind::{InvalidData, NotFound, Other}, + Read, Result, Seek, SeekFrom, Write, + }, + os::unix::{ + fs::{symlink, MetadataExt}, + prelude::PermissionsExt, + process::CommandExt, + }, + path::{Path, PathBuf}, + process::{exit, Command}, + str, + thread::{sleep, spawn}, + time::{self, Duration, Instant}, }; -use which::which; use cfg_if::cfg_if; -use xxhash_rust::xxh3::xxh3_64; use goblin::elf::{Elf, SectionHeader}; use memfd_exec::{MemFdExecutable, Stdio}; -use nix::{libc, sys::{wait::waitpid, signal::{Signal, kill}}}; -use nix::unistd::{access, fork, setsid, getcwd, AccessFlags, ForkResult, Pid}; -use signal_hook::{consts::{SIGINT, SIGTERM, SIGQUIT, SIGHUP}, iterator::Signals}; +use nix::fcntl::{open, OFlag}; +use nix::unistd::{access, close, fork, getcwd, setsid, AccessFlags, ForkResult, Pid}; +use nix::{ + errno::Errno, + libc, + mount::umount, + sys::{ + signal::{kill, Signal}, + stat::Mode, + wait::waitpid, + }, +}; +use signal_hook::{ + consts::{SIGHUP, SIGINT, SIGQUIT, SIGTERM, SIGUSR1, SIGUSR2}, + iterator::Signals, +}; +use which::which_all; +use xxhash_rust::xxh3::xxh3_64; + +mod elf_layout; +const _LINUX_CAPABILITY_VERSION_3: u32 = 0x20080522; +const SECBIT_NOROOT: libc::c_ulong = 1; +const SECBIT_NOROOT_LOCKED: libc::c_ulong = 2; const URUNTIME_VERSION: &str = env!("CARGO_PKG_VERSION"); + const URUNTIME_MOUNT: &str = "URUNTIME_MOUNT=3"; const URUNTIME_CLEANUP: &str = "URUNTIME_CLEANUP=1"; const URUNTIME_EXTRACT: &str = "URUNTIME_EXTRACT=3"; +const URUNTIME_UNSHARE: &str = "URUNTIME_UNSHARE=0"; + const REUSE_CHECK_DELAY: &str = "5s"; const MAX_EXTRACT_SELF_SIZE: u64 = 350 * 1024 * 1024; // 350 MB #[cfg(feature = "dwarfs")] @@ -35,13 +68,40 @@ const DWARFS_READAHEAD: &str = "32M"; cfg_if! { if #[cfg(feature = "appimage")] { const ARG_PFX: &str = "appimage"; + const ENV_NAME: &str = "APPIMAGE"; const SELF_NAME: &str = "AppImage"; } else { const ARG_PFX: &str = "runtime"; + const ENV_NAME: &str = "RUNIMAGE"; const SELF_NAME: &str = "RunImage"; } } +macro_rules! get_env_var { + ($var:literal) => { + std::env::var($var).unwrap_or_default() + }; + ($($arg:tt)*) => { + { + let env_name = format!($($arg)*); + std::env::var(env_name).unwrap_or_default() + } + }; +} + +#[repr(C)] +struct CapHeader { + version: u32, + pid: i32, +} +#[repr(C)] +#[derive(Clone, Copy)] +struct CapData { + effective: u32, + permitted: u32, + inheritable: u32, +} + #[derive(Debug)] struct Runtime { path: PathBuf, @@ -61,121 +121,121 @@ struct Image { #[derive(Debug)] struct Embed { #[cfg(feature = "squashfs")] - squashfuse: Vec, + squashfuse: &'static [u8], #[cfg(feature = "squashfs")] - unsquashfs: Vec, + unsquashfs: &'static [u8], #[cfg(all(not(feature = "lite"), feature = "squashfs"))] - mksquashfs: Vec, + mksquashfs: &'static [u8], #[cfg(feature = "dwarfs")] - dwarfs_universal: Vec, + dwarfs_universal: &'static [u8], } impl Embed { fn new() -> Self { - cfg_if! { - if #[cfg(feature = "upx")] { - Embed { - #[cfg(feature = "squashfs")] - squashfuse: include_bytes!("../assets/squashfuse-upx").to_vec(), - #[cfg(feature = "squashfs")] - unsquashfs: include_bytes!("../assets/unsquashfs-upx").to_vec(), - #[cfg(all(not(feature = "lite"), feature = "squashfs"))] - mksquashfs: include_bytes!("../assets/mksquashfs-upx").to_vec(), - #[cfg(all(feature = "lite", feature = "dwarfs"))] - dwarfs_universal: include_bytes!("../assets/dwarfs-fuse-extract-upx").to_vec(), - #[cfg(all(not(feature = "lite"), feature = "dwarfs"))] - dwarfs_universal: include_bytes!("../assets/dwarfs-universal-upx").to_vec(), - } - } else { - Embed { - #[cfg(feature = "squashfs")] - squashfuse: include_bytes!("../assets/squashfuse-zst").to_vec(), - #[cfg(feature = "squashfs")] - unsquashfs: include_bytes!("../assets/unsquashfs-zst").to_vec(), - #[cfg(all(not(feature = "lite"), feature = "squashfs"))] - mksquashfs: include_bytes!("../assets/mksquashfs-zst").to_vec(), - #[cfg(all(feature = "lite", feature = "dwarfs"))] - dwarfs_universal: include_bytes!("../assets/dwarfs-fuse-extract-zst").to_vec(), - #[cfg(all(not(feature = "lite"), feature = "dwarfs"))] - dwarfs_universal: include_bytes!("../assets/dwarfs-universal-zst").to_vec(), - } - } + Embed { + #[cfg(feature = "squashfs")] + squashfuse: include_bytes!(concat!(env!("URUNTIME_HELPER_DIR"), "/squashfuse-zst")), + #[cfg(feature = "squashfs")] + unsquashfs: include_bytes!(concat!(env!("URUNTIME_HELPER_DIR"), "/unsquashfs-zst")), + #[cfg(all(not(feature = "lite"), feature = "squashfs"))] + mksquashfs: include_bytes!(concat!(env!("URUNTIME_HELPER_DIR"), "/mksquashfs-zst")), + #[cfg(all(feature = "lite", feature = "dwarfs"))] + dwarfs_universal: include_bytes!(concat!( + env!("URUNTIME_HELPER_DIR"), + "/dwarfs-fuse-extract-zst" + )), + #[cfg(all(not(feature = "lite"), feature = "dwarfs"))] + dwarfs_universal: include_bytes!(concat!( + env!("URUNTIME_HELPER_DIR"), + "/dwarfs-universal-zst" + )), } } #[cfg(feature = "squashfs")] fn squashfuse(&self, exec_args: Vec) { - mfd_exec("squashfuse", &self.squashfuse, exec_args); + mfd_exec("squashfuse", self.squashfuse, exec_args); } #[cfg(feature = "squashfs")] fn unsquashfs(&self, exec_args: Vec) { - mfd_exec("unsquashfs", &self.unsquashfs, exec_args); + mfd_exec("unsquashfs", self.unsquashfs, exec_args); } #[cfg(feature = "squashfs")] fn sqfscat(&self, exec_args: Vec) { - mfd_exec("sqfscat", &self.unsquashfs, exec_args); + mfd_exec("sqfscat", self.unsquashfs, exec_args); } #[cfg(all(not(feature = "lite"), feature = "squashfs"))] fn mksquashfs(&self, exec_args: Vec) { - mfd_exec("mksquashfs", &self.mksquashfs, exec_args); + mfd_exec("mksquashfs", self.mksquashfs, exec_args); } #[cfg(all(not(feature = "lite"), feature = "squashfs"))] fn sqfstar(&self, exec_args: Vec) { - mfd_exec("sqfstar", &self.mksquashfs, exec_args); + mfd_exec("sqfstar", self.mksquashfs, exec_args); } #[cfg(feature = "dwarfs")] fn dwarfs(&self, exec_args: Vec) { - mfd_exec("dwarfs", &self.dwarfs_universal, exec_args); + mfd_exec("dwarfs", self.dwarfs_universal, exec_args); } #[cfg(all(not(feature = "lite"), feature = "dwarfs"))] fn dwarfsck(&self, exec_args: Vec) { - mfd_exec("dwarfsck", &self.dwarfs_universal, exec_args); + mfd_exec("dwarfsck", self.dwarfs_universal, exec_args); } #[cfg(all(not(feature = "lite"), feature = "dwarfs"))] fn mkdwarfs(&self, exec_args: Vec) { - mfd_exec("mkdwarfs", &self.dwarfs_universal, exec_args); + mfd_exec("mkdwarfs", self.dwarfs_universal, exec_args); } #[cfg(feature = "dwarfs")] fn dwarfsextract(&self, exec_args: Vec) { - mfd_exec("dwarfsextract", &self.dwarfs_universal, exec_args); + mfd_exec("dwarfsextract", self.dwarfs_universal, exec_args); } } fn mfd_exec(exec_name: &str, exec_bytes: &[u8], exec_args: Vec) { env::set_var("LC_ALL", "C"); - if get_env_var("MALLOC_CONF").is_empty() { - env::set_var("MALLOC_CONF", "background_thread:true,dirty_decay_ms:1000,muzzy_decay_ms:1000") + if get_env_var!("MALLOC_CONF").is_empty() { + env::set_var( + "MALLOC_CONF", + "background_thread:true,dirty_decay_ms:1000,muzzy_decay_ms:1000", + ) } - #[cfg(not(feature = "upx"))] fn decompress(exec_name: &str, data: &[u8]) -> Vec { if exec_name != "uruntime" { - let mut decoder = zstd::stream::read::Decoder::new(data).unwrap(); + let mut decoder = zstd::stream::read::Decoder::new(data).unwrap_or_else(|err| { + eprintln!("Failed to create decoder for decompress embed exe: {exec_name}: {err}"); + exit(1) + }); let mut decompressed_data = Vec::new(); - decoder.read_to_end(&mut decompressed_data).unwrap(); + decoder + .read_to_end(&mut decompressed_data) + .unwrap_or_else(|err| { + eprintln!("Failed to decompress embed exe: {exec_name}: {err}"); + exit(1) + }); decompressed_data - } else { data.to_vec() } + } else { + data.to_vec() + } } - #[cfg(not(feature = "upx"))] let exec_bytes = &decompress(exec_name, exec_bytes); let err = MemFdExecutable::new(exec_name, exec_bytes) .args(exec_args) - .envs(env::vars()) + .envs(env::vars_os()) .exec(Stdio::inherit()); eprintln!("Failed to execute {exec_name}: {err}"); exit(1) } -fn get_image(path: &PathBuf, offset: u64) -> Result { +fn get_image(path: &Path, offset: u64) -> Result { let mut file = File::open(path)?; let mut buff = [0u8; 4]; file.seek(SeekFrom::Start(offset))?; @@ -184,91 +244,533 @@ fn get_image(path: &PathBuf, offset: u64) -> Result { path: path.to_path_buf(), offset, is_dwar: false, - is_squash: false + is_squash: false, }; if bytes_read == 4 { - let read_str = String::from_utf8_lossy(&buff); - if read_str.contains("DWAR") { + if buff == *b"DWAR" { image.is_dwar = true - } else if read_str.contains("hsqs") { + } else if buff == *b"hsqs" { image.is_squash = true } } if !image.is_squash && !image.is_dwar { - return Err(Error::new(NotFound, "SquashFS or DwarFS image not found!")) + return Err(Error::new(NotFound, "SquashFS or DwarFS image not found!")); } Ok(image) } -fn get_env_var(var: &str) -> String { - env::var(var).unwrap_or("".into()) -} - fn add_to_path(path: &PathBuf) { - let old_path = get_env_var("PATH"); + let old_path = get_env_var!("PATH"); if old_path.is_empty() { env::set_var("PATH", path) } else { - let new_path = path.to_str().unwrap(); + let new_path = path.to_str().unwrap_or_default(); if !old_path.contains(new_path) { env::set_var("PATH", format!("{new_path}:{old_path}")) } } } -fn check_fuse() -> bool { - let mut is_fusermount = true; - let tmp_path_dir = &PathBuf::from("/tmp/.path"); - if tmp_path_dir.is_dir() { - add_to_path(tmp_path_dir) +fn restore_capabilities() { + let mut caps = CapHeader { + version: _LINUX_CAPABILITY_VERSION_3, + pid: 0, + }; + let mut cap_data = [CapData { + effective: 0, + permitted: 0, + inheritable: 0, + }; 2]; + if unsafe { libc::syscall(libc::SYS_capget, &mut caps, cap_data.as_mut_ptr()) } == 0 { + let last_cap = last_capability(); + let all_caps = if last_cap == 63 { + u64::MAX + } else { + (1u64 << (last_cap + 1)) - 1 + }; + cap_data[0].effective = all_caps as u32; + cap_data[0].permitted = all_caps as u32; + cap_data[0].inheritable = all_caps as u32; + cap_data[1].effective = (all_caps >> 32) as u32; + cap_data[1].permitted = (all_caps >> 32) as u32; + cap_data[1].inheritable = (all_caps >> 32) as u32; + unsafe { libc::syscall(libc::SYS_capset, &caps, cap_data.as_ptr()) }; + for cap in 0..=last_cap { + unsafe { libc::prctl(libc::PR_CAP_AMBIENT, libc::PR_CAP_AMBIENT_RAISE, cap, 0, 0) }; + } + } else { + eprintln!( + "Warning: failed to get capabilities: {}", + Error::last_os_error() + ) } - let fusermount_prog = &get_env_var("FUSERMOUNT_PROG"); - if PathBuf::from(fusermount_prog).is_file() { +} + +fn last_capability() -> u32 { + std::fs::read_to_string("/proc/sys/kernel/cap_last_cap") + .ok() + .and_then(|s| s.trim().parse::().ok()) + .unwrap_or(39) + .min(63) +} + +fn should_drop_capabilities(unshare_succeeded: bool, drop_caps: bool) -> bool { + unshare_succeeded && drop_caps +} + +fn embedded_unshare_policy(mode: &str) -> (bool, bool, bool) { + match mode { + "=1" => (true, false, false), + "=2" => (true, true, false), + "=3" => (false, false, true), + _ => (false, false, false), + } +} + +fn fallback_should_drop_capabilities( + fallback_unshare_succeeded: bool, + drop_caps_on_fallback: bool, +) -> bool { + fallback_unshare_succeeded && drop_caps_on_fallback +} + +fn environment_drop_caps_policy(value: &str) -> (bool, bool, bool) { + match value { + "1" => (true, false, false), + "2" => (true, true, false), + "3" => (false, false, true), + _ => (false, false, false), + } +} + +fn is_runtime_option(arg: &str, option: &str) -> bool { + arg.strip_prefix("--") + .and_then(|arg| arg.strip_prefix(ARG_PFX)) + .is_some_and(|arg| arg.strip_prefix('-') == Some(option)) +} + +#[derive(Debug, Default, Eq, PartialEq)] +struct UnshareCliOptions { + enable: bool, + root: bool, + uid: Option, + gid: Option, + drop_caps: bool, + drop_caps_on_fallback: bool, +} + +fn parse_unshare_cli_options( + args: &mut Vec, + prefix: &str, +) -> std::result::Result { + let base = format!("--{prefix}-unshare"); + if !args + .iter() + .take_while(|arg| arg.as_str() != "--") + .any(|arg| arg.starts_with(&base)) + { + return Ok(UnshareCliOptions::default()); + } + let mut options = UnshareCliOptions::default(); + let mut retained = Vec::with_capacity(args.len()); + let mut input = args.clone().into_iter(); + while let Some(arg) = input.next() { + if arg == "--" { + retained.push(arg); + retained.extend(input); + break; + } + let suffix = arg.strip_prefix(&base); + let value_option = match suffix { + Some("-uid") => Some(("uid", input.next())), + Some(suffix) if suffix.starts_with("-uid=") => { + Some(("uid", Some(suffix[5..].to_string()))) + } + Some("-gid") => Some(("gid", input.next())), + Some(suffix) if suffix.starts_with("-gid=") => { + Some(("gid", Some(suffix[5..].to_string()))) + } + _ => None, + }; + if let Some((kind, value)) = value_option { + let value = value.ok_or_else(|| format!("{arg} requires a numeric value"))?; + value + .parse::() + .map_err(|_| format!("invalid {kind} value `{value}` for {arg}"))?; + if kind == "uid" { + options.uid = Some(value); + } else { + options.gid = Some(value); + } + options.enable = true; + continue; + } + if suffix == Some("") { + options.enable = true; + } else if suffix == Some("-root") { + options.enable = true; + options.root = true; + } else if suffix == Some("-drop-caps") { + options.enable = true; + options.drop_caps = true; + } else if suffix == Some("-fallback-drop-caps") { + options.drop_caps_on_fallback = true; + } else { + retained.push(arg); + } + } + if options.drop_caps_on_fallback && options.enable { + options.drop_caps = true; + options.drop_caps_on_fallback = false; + } + *args = retained; + Ok(options) +} + +fn remove_runtime_separator(args: &mut Vec) { + if let Some(index) = args.iter().position(|arg| arg == "--") { + args.remove(index); + } +} + +fn drop_capabilities(last_cap: u32) -> Result<()> { + let syscall_failed = |result: libc::c_long| { + if result == 0 { + Ok(()) + } else { + Err(Error::last_os_error()) + } + }; + + let mut caps = CapHeader { + version: _LINUX_CAPABILITY_VERSION_3, + pid: 0, + }; + let mut current = [CapData { + effective: 0, + permitted: 0, + inheritable: 0, + }; 2]; + syscall_failed(unsafe { libc::syscall(libc::SYS_capget, &mut caps, current.as_mut_ptr()) })?; + let securebits = unsafe { libc::prctl(libc::PR_GET_SECUREBITS, 0, 0, 0, 0) }; + let capabilities_are_empty = current + .iter() + .all(|data| data.effective == 0 && data.permitted == 0 && data.inheritable == 0); + if capabilities_are_empty + && securebits >= 0 + && securebits as libc::c_ulong & (SECBIT_NOROOT | SECBIT_NOROOT_LOCKED) + == SECBIT_NOROOT | SECBIT_NOROOT_LOCKED + { + return Ok(()); + } + + syscall_failed(unsafe { + libc::prctl( + libc::PR_CAP_AMBIENT, + libc::PR_CAP_AMBIENT_CLEAR_ALL, + 0, + 0, + 0, + ) as libc::c_long + })?; + syscall_failed(unsafe { + libc::prctl( + libc::PR_SET_SECUREBITS, + SECBIT_NOROOT | SECBIT_NOROOT_LOCKED, + 0, + 0, + 0, + ) as libc::c_long + })?; + for cap in 0..=last_cap { + syscall_failed(unsafe { + libc::prctl(libc::PR_CAPBSET_DROP, cap, 0, 0, 0) as libc::c_long + })?; + } + let cap_data = [CapData { + effective: 0, + permitted: 0, + inheritable: 0, + }; 2]; + syscall_failed(unsafe { libc::syscall(libc::SYS_capset, &caps, cap_data.as_ptr()) }) +} + +fn try_make_mount_private() -> bool { + unsafe { + libc::mount( + c"none".as_ptr(), + c"/".as_ptr(), + c"none".as_ptr(), + libc::MS_REC | libc::MS_PRIVATE, + std::ptr::null(), + ) == 0 + } +} + +fn try_unshare(uid: u32, gid: u32, unshare_uid: &str, unshare_gid: &str) -> bool { + let target_uid = unshare_uid.parse().unwrap_or(uid); + let target_gid = unshare_gid.parse().unwrap_or(gid); + let flags = libc::CLONE_NEWUSER | libc::CLONE_NEWNS; + let result = unsafe { libc::unshare(flags) }; + if result == 0 { + let _ = fs::write("/proc/self/setgroups", "deny"); + let uid_map = format!("{target_uid} {uid} 1"); + let gid_map = format!("{target_gid} {gid} 1"); + if fs::write("/proc/self/uid_map", uid_map).is_ok() + && fs::write("/proc/self/gid_map", gid_map).is_ok() + { + restore_capabilities(); + if !try_make_mount_private() { + eprintln!( + "Warning: failed to make mount private: {}", + Error::last_os_error() + ) + } + return true; + } + } + eprintln!( + "Failed to create user and mount namespaces: {}", + Error::last_os_error() + ); + false +} + +fn is_in_user_and_mount_namespace() -> bool { + let uid_map = match read_to_string("/proc/self/uid_map") { + Ok(content) => content, + Err(_) => return false, + }; + let uid_map = uid_map.trim(); + if uid_map.is_empty() + || uid_map.split_whitespace().collect::>() == vec!["0", "0", "4294967295"] + { + return false; + } + let lines: Vec<&str> = uid_map.lines().collect(); + if lines.is_empty() { + return false; + } + for line in lines { + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() == 3 { + if let Ok(count) = parts[2].parse::() { + if count < 4294967295 { + return try_make_mount_private(); + } + } + } + } + false +} + +fn namespace_diff_flags(pid: Pid) -> Result { + let mut flags = 0; + for (namespace, flag) in [("user", libc::CLONE_NEWUSER), ("mnt", libc::CLONE_NEWNS)] { + let current = fs::read_link(format!("/proc/self/ns/{namespace}"))?; + let target = fs::read_link(format!("/proc/{pid}/ns/{namespace}"))?; + if current != target { + flags |= flag + } + } + Ok(flags) +} + +fn try_setns(pid: Pid) -> bool { + let flags = match namespace_diff_flags(pid) { + Ok(0) => return true, + Ok(flags) => flags, + Err(err) => { + eprintln!("Failed to compare namespaces: {err}"); + return false; + } + }; + let original_cwd = getcwd().ok(); + let pidfd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid.as_raw() as i64, 0i64) as i32 }; + if pidfd >= 0 { + let result = unsafe { libc::setns(pidfd, flags) }; + let _ = close(pidfd); + if result == 0 { + if flags & libc::CLONE_NEWUSER != 0 { + restore_capabilities() + } + if flags & libc::CLONE_NEWNS != 0 { + if !try_make_mount_private() { + eprintln!( + "Warning: failed to make mount private: {}", + Error::last_os_error() + ) + } + if let Some(cwd) = original_cwd { + if let Err(err) = env::set_current_dir(&cwd) { + eprintln!("Warning: failed to restore working directory: {err}"); + } + } + } + return true; + } + eprintln!( + "Failed to enter namespaces via pidfd: {}", + Error::last_os_error() + ); + return false; + } + eprintln!( + "Failed to open pidfd: {} - mount point reuse unavailable", + Error::last_os_error() + ); + false +} + +fn read_mount_pid_file(mount_point: &Path, extension: &str) -> Option { + let pid_file = mount_point.with_extension(extension); + if let Ok(pid_str) = read_to_string(&pid_file) { + if let Ok(pid) = pid_str.trim().parse::() { + return Some(Pid::from_raw(pid)); + } + } + None +} + +fn write_mount_pid_file(mount_point: &Path, pid: Pid, unshare_succeeded: bool) -> Result<()> { + let pidfile_ext = if !unshare_succeeded { "pid" } else { "un.pid" }; + let pidfile_path = mount_point.with_extension(pidfile_ext); + fs::write(&pidfile_path, pid.as_raw().to_string())?; + Ok(()) +} + +fn try_reuse_unshare_mount_point(mount_point: &Path) -> Option { + let pid = read_mount_pid_file(mount_point, "un.pid")?; + if !is_pid_exists(pid) { + let un_pid_file = mount_point.with_extension("un.pid"); + let _ = remove_file(&un_pid_file); + return None; + } + if is_mounted(mount_point).unwrap_or(false) { + return Some(pid); + } + if try_setns(pid) && is_mounted(mount_point).unwrap_or(false) { + return Some(pid); + } + None +} + +fn check_fuse( + uruntime: &Path, + uid: u32, + gid: u32, + unshare_uid: &str, + unshare_gid: &str, + unshare_succeeded: &mut bool, + is_unshare: &mut bool, +) -> bool { + if access("/dev/fuse", AccessFlags::R_OK | AccessFlags::W_OK).is_err() { + return false; + } + if uid == 0 || *unshare_succeeded || is_in_user_and_mount_namespace() { + return true; + } + fn create_fusermount_dir(tmp_path_dir: &PathBuf) -> bool { if !tmp_path_dir.is_dir() { if let Err(err) = create_dir_all(tmp_path_dir) { - eprintln!("Failed to create fusermount PATH dir: {err}: {:?}", tmp_path_dir); - exit(1) + eprintln!( + "Failed to create fusermount PATH dir: {err}: {:?}", + tmp_path_dir + ); + return false; } add_to_path(tmp_path_dir); } - let fsmntlink_path = tmp_path_dir.join(basename(fusermount_prog)); + true + } + fn create_fusermount_symlink( + tmp_path_dir: &Path, + fusermount_path: &str, + fusermount_name: &str, + ) -> bool { + let fsmntlink_path = tmp_path_dir.join(fusermount_name); let _ = remove_file(&fsmntlink_path); - if let Err(err) = symlink(fusermount_prog, &fsmntlink_path) { - eprintln!("Failed to create fusermount symlink: {err}: {:?}", fsmntlink_path); + if let Err(err) = symlink(fusermount_path, &fsmntlink_path) { + eprintln!( + "Failed to create fusermount symlink: {err}: {:?}", + fsmntlink_path + ); + return false; + } + true + } + let mut is_fusermount = true; + let fusermount_list = ["fusermount", "fusermount3"]; + let tmp_path_dir = &PathBuf::from(format!("/tmp/.path{uid}")); + if tmp_path_dir.is_dir() { + let uruntime_path = uruntime.canonicalize().ok(); + for fusermount in fusermount_list { + let old_symlink = tmp_path_dir.join(fusermount); + if old_symlink.exists() { + if let Ok(canonical_path) = old_symlink.canonicalize() { + if is_suid_exe(&canonical_path).unwrap_or(false) { + continue; + } + if let Some(ref uruntime_canonical) = uruntime_path { + if canonical_path == *uruntime_canonical { + continue; + } + } + } + let _ = remove_file(&old_symlink); + } + } + add_to_path(tmp_path_dir) + } + let fusermount_prog = &get_env_var!("FUSERMOUNT_PROG"); + if is_suid_exe(&PathBuf::from(fusermount_prog)).unwrap_or(false) { + if !create_fusermount_dir(tmp_path_dir) { + exit(1) + } + if !create_fusermount_symlink(tmp_path_dir, fusermount_prog, basename(fusermount_prog)) { exit(1) } } else { - for fusermount in ["fusermount", "fusermount3"] { + for fusermount in fusermount_list { + if find_suid_exe(fusermount).is_some() { + continue; + } let fallback: &str = if fusermount.ends_with("3") { "fusermount" } else { "fusermount3" }; - if which(fusermount).is_err() { - if let Ok(fusermount_path) = which(fallback) { - if !tmp_path_dir.is_dir() { - if let Err(err) = create_dir_all(tmp_path_dir) { - eprintln!("Failed to create fusermount fallback dir: {err}: {:?}", tmp_path_dir); - break - } - } - let fsmntlink_path = tmp_path_dir.join(fusermount); - let _ = remove_file(&fsmntlink_path); - if let Err(err) = symlink(fusermount_path, &fsmntlink_path) { - eprintln!("Failed to create fusermount fallback symlink: {err}: {:?}", tmp_path_dir); - break - } - add_to_path(tmp_path_dir); - break - } else { - is_fusermount = false + if let Some(fusermount_path) = find_suid_exe(fallback) { + if !create_fusermount_dir(tmp_path_dir) { + break; + } + if !create_fusermount_symlink( + tmp_path_dir, + &fusermount_path.to_string_lossy(), + fusermount, + ) { + break; } + break; } + is_fusermount = false } } - if access("/dev/fuse", AccessFlags::R_OK).is_err() || - access("/dev/fuse", AccessFlags::W_OK).is_err() || !is_fusermount { - return false + if !is_fusermount { + eprintln!("SUID fusermount not found in PATH, trying to unshare..."); + *is_unshare = true; + if try_unshare(uid, gid, unshare_uid, unshare_gid) { + *unshare_succeeded = true; + return true; + } + for fusermount in fusermount_list { + if !create_fusermount_dir(tmp_path_dir) { + break; + } + if !create_fusermount_symlink(tmp_path_dir, &uruntime.to_string_lossy(), fusermount) { + break; + } + } } true } @@ -280,61 +782,73 @@ macro_rules! check_extract { $self_exe:expr, $true_block:block ) => { - eprintln!("{}: failed to utilize FUSE during startup!", basename($self_exe.to_str().unwrap())); + eprintln!( + "{}: failed to utilize FUSE during startup!", + basename($self_exe.to_str().unwrap_or_default()) + ); let self_size = get_file_size($self_exe).unwrap_or_else(|err| { eprintln!("Failed to get self size: {err}"); exit(1) }); - if !$is_mount_only && ($uruntime_extract == 2 || ($uruntime_extract == 3 && - self_size <= MAX_EXTRACT_SELF_SIZE)) { + if !$is_mount_only + && ($uruntime_extract == 2 + || ($uruntime_extract == 3 && self_size <= MAX_EXTRACT_SELF_SIZE)) + { $true_block } else { eprintln!( -"Cannot mount {SELF_NAME}, please check your FUSE setup. + "Cannot mount {SELF_NAME}, please check your FUSE setup. You might still be able to extract the contents of this {SELF_NAME} if you run it with the --{ARG_PFX}-extract option See https://github.com/AppImage/AppImageKit/wiki/FUSE -and run it with the --{ARG_PFX}-help option for more information"); +and run it with the --{ARG_PFX}-help option for more information" + ); exit(1) } }; } fn get_section_index(elf: &Elf<'_>, section_name: &str) -> Result { - let section_index = elf.section_headers + let section_index = elf + .section_headers .iter() .position(|sh| { - if let Some(name) = elf.shdr_strtab.get_at(sh.sh_name) - { name == section_name } else { false } + if let Some(name) = elf.shdr_strtab.get_at(sh.sh_name) { + name == section_name + } else { + false + } }) - .ok_or(Error::new(InvalidData, - format!("Section header with name '{section_name}' not found!") + .ok_or(Error::new( + InvalidData, + format!("Section header with name '{section_name}' not found!"), ))?; Ok(section_index) } fn get_section_header(headers_bytes: &[u8], section_name: &str) -> Result { - let elf = Elf::parse(headers_bytes) - .map_err(|err| Error::new(InvalidData, err))?; + let elf = Elf::parse(headers_bytes).map_err(|err| Error::new(InvalidData, err))?; let section_index = get_section_index(&elf, section_name)?; Ok(elf.section_headers[section_index].clone()) } fn get_section_data(headers_bytes: &[u8], section_name: &str) -> Result { let section = &mut get_section_header(headers_bytes, section_name)?; - let section_data = &headers_bytes[section.sh_offset as usize..(section.sh_offset + section.sh_size) as usize]; + let section_data = + &headers_bytes[section.sh_offset as usize..(section.sh_offset + section.sh_size) as usize]; if let Ok(data_str) = str::from_utf8(section_data) { Ok(data_str.trim().trim_matches('\0').into()) } else { - Err(Error::new(InvalidData, - format!("Section data is not valid UTF-8: {section_name}") + Err(Error::new( + InvalidData, + format!("Section data is not valid UTF-8: {section_name}"), )) } } fn add_section_data(runtime: &Runtime, section_name: &str, exec_args: &[String]) -> Result<()> { - if get_env_var(&format!("TARGET_{}", SELF_NAME.to_uppercase())).is_empty() { - env::set_var(format!("TARGET_{}", SELF_NAME.to_uppercase()), &runtime.path); + if get_env_var!("TARGET_{}", ENV_NAME).is_empty() { + env::set_var(format!("TARGET_{ENV_NAME}"), &runtime.path); mfd_exec("uruntime", &runtime.headers_bytes, exec_args.to_vec()); } let section = get_section_header(&runtime.headers_bytes, section_name)?; @@ -348,16 +862,17 @@ fn add_section_data(runtime: &Runtime, section_name: &str, exec_args: &[String]) } else { section_data.as_bytes() } - } else { &[] }; + } else { + &[] + }; let new_size = string_bytes.len() as u64; if new_size > original_size { - return Err(Error::new(InvalidData, - "New section header data is larger than the section size!" + return Err(Error::new( + InvalidData, + "New section header data is larger than the section size!", )); } - let mut file = fs::OpenOptions::new() - .write(true) - .open(&runtime.path)?; + let mut file = fs::OpenOptions::new().write(true).open(&runtime.path)?; file.seek(SeekFrom::Start(offset))?; file.write_all(string_bytes)?; if new_size < original_size { @@ -370,40 +885,35 @@ fn add_section_data(runtime: &Runtime, section_name: &str, exec_args: &[String]) fn get_runtime(path: &PathBuf) -> Result { let mut file = File::open(path)?; - let mut elf_header_raw = [0; 64]; - file.read_exact(&mut elf_header_raw)?; - let section_table_offset = u64::from_le_bytes(elf_header_raw[40..48].try_into().unwrap()); // e_shoff - let section_count = u16::from_le_bytes(elf_header_raw[60..62].try_into().unwrap()); // e_shnum - let section_table_size = section_count as u64 * 64; - let required_bytes = section_table_offset + section_table_size; - let mut headers_bytes = vec![0; required_bytes as usize]; - file.seek(SeekFrom::Start(0))?; - file.read_exact(&mut headers_bytes)?; - let elf = Elf::parse(&headers_bytes) - .map_err(|err| Error::new(InvalidData, err))?; - let section_table_end = - elf.header.e_shoff + (elf.header.e_shentsize as u64 * elf.header.e_shnum as u64); - let last_section_end = elf - .section_headers - .last() - .map(|section| section.sh_offset + section.sh_size) - .unwrap_or(0); + let file_len = file.metadata()?.len(); + let prefix = elf_layout::read_elf_prefix(&mut file, file_len)?; + let headers_bytes = prefix.bytes; + let elf = Elf::parse(&headers_bytes).map_err(|err| Error::new(InvalidData, err))?; let envs = if let Ok(section_index) = get_section_index(&elf, ".envs") { let section = &elf.section_headers[section_index]; - let section_data = &headers_bytes[section.sh_offset as usize..(section.sh_offset + section.sh_size) as usize]; - str::from_utf8(section_data).unwrap_or_default().trim_matches('\0').to_string() - } else { "".into() }; + let section_data = &headers_bytes + [section.sh_offset as usize..(section.sh_offset + section.sh_size) as usize]; + str::from_utf8(section_data) + .unwrap_or_default() + .trim_matches('\0') + .to_string() + } else { + "".into() + }; Ok(Runtime { path: path.to_path_buf(), headers_bytes, - size: section_table_end.max(last_section_end), + size: prefix.boundary, envs, }) } fn random_string(length: usize) -> String { const CHARSET: &[u8] = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - let mut rng = time::SystemTime::now().duration_since(time::UNIX_EPOCH).unwrap().as_millis(); + let mut rng = time::SystemTime::now() + .duration_since(time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); let mut result = String::with_capacity(length); for _ in 0..length { rng = rng.wrapping_mul(48271).wrapping_rem(0x7FFFFFFF); @@ -413,32 +923,99 @@ fn random_string(length: usize) -> String { result } -fn basename(path: &str) -> String { - let pieces: Vec<&str> = path.rsplit('/').collect(); - pieces.first().unwrap().to_string() +fn basename(path: &str) -> &str { + path.rsplit('/').next().unwrap_or_default() } -fn is_mount_point(path: &PathBuf) -> Result { - let metadata = fs::metadata(path)?; +fn is_broken_mount_errno(err: Errno) -> bool { + err == Errno::ENOTCONN || err == Errno::ESTALE || err == Errno::EIO +} + +fn get_metadata_or_broken_mount(path: &Path) -> Result { + match fs::metadata(path) { + Ok(m) => Ok(m), + Err(err) => { + if let Some(errno) = Errno::from_raw(err.raw_os_error().unwrap_or(0)).into() { + if is_broken_mount_errno(errno) { + return Err(Error::new(Other, "broken mount point")); + } + } + Err(err) + } + } +} + +fn is_mount_point(path: &Path) -> Result { + let full_path = &path.canonicalize().unwrap_or(path.into()); + let metadata = match get_metadata_or_broken_mount(full_path) { + Ok(m) => m, + Err(err) if err.kind() == Other => return Ok(true), + Err(err) => return Err(err), + }; let device_id = metadata.dev(); - match path.parent() { + match full_path.parent() { Some(parent) => { - let parent_metadata = fs::metadata(parent)?; + let parent_metadata = match get_metadata_or_broken_mount(parent) { + Ok(m) => m, + Err(err) if err.kind() == Other => return Ok(true), + Err(err) => return Err(err), + }; Ok(device_id != parent_metadata.dev()) } - None => Ok(false) + None => Ok(false), + } +} + +fn is_mounted(path: &Path) -> Result { + let is_mount = is_mount_point(path)?; + if is_mount { + let path = &path.canonicalize().unwrap_or(path.into()); + match open( + path, + OFlag::O_RDONLY | OFlag::O_DIRECTORY | OFlag::O_CLOEXEC, + Mode::empty(), + ) { + Ok(fd) => { + let _ = close(fd); + } + Err(err) => { + if is_broken_mount_errno(err) { + try_unmount(None, path); + return Ok(false); + } + return Err(Error::from(err)); + } + } } + Ok(is_mount) } fn get_file_size(path: &PathBuf) -> Result { Ok(fs::metadata(path)?.len()) } +fn is_suid_exe(path: &PathBuf) -> Result { + let metadata = fs::metadata(path)?; + let permissions = metadata.permissions(); + let mode = permissions.mode(); + Ok((mode & 0o4000 != 0) && (mode & 0o111 != 0)) +} + +fn find_suid_exe(name: &str) -> Option { + for path in which_all(name).ok()? { + let canonical_path = path.canonicalize().unwrap_or(path); + if is_suid_exe(&canonical_path).unwrap_or(false) { + return Some(canonical_path); + } + } + None +} + fn is_dir_inuse(mount_point: &PathBuf) -> Result { for entry in fs::read_dir("/proc")? { if let Ok(target) = fs::read_link(entry?.path().join("exe")) { if target.starts_with(mount_point) { - return Ok(true) + return Ok(true); } } } @@ -449,7 +1026,8 @@ fn wait_dir_notuse( mount_point: &PathBuf, timeout: Option, delay: Option, - delay_check: bool) -> bool { + delay_check: bool, +) -> bool { let start_time = Instant::now(); let default_delay = Duration::from_millis(100); let delay = delay.unwrap_or(default_delay); @@ -457,17 +1035,22 @@ fn wait_dir_notuse( if delay_check { sleep(delay); for num_check in 1..=5 { - if is_dir_inuse(mount_point).unwrap_or(false) - { break } else { sleep(default_delay * 2) } - if num_check == 5 { return true } + if is_dir_inuse(mount_point).unwrap_or(false) { + break; + } else { + sleep(default_delay * 2) + } + if num_check == 5 { + return true; + } } } else { if !is_dir_inuse(mount_point).unwrap_or(false) { - return true + return true; } if let Some(timeout) = timeout { if start_time.elapsed() >= timeout { - return false + return false; } } } @@ -476,10 +1059,7 @@ fn wait_dir_notuse( } fn is_pid_exists(pid: Pid) -> bool { - if PathBuf::from(format!("/proc/{pid}")).exists() { - return true - } - false + PathBuf::from(format!("/proc/{pid}")).exists() } fn wait_pid_exit(pid: Pid, timeout: Option) -> bool { @@ -487,7 +1067,7 @@ fn wait_pid_exit(pid: Pid, timeout: Option) -> bool { while is_pid_exists(pid) { if let Some(timeout) = timeout { if start_time.elapsed() >= timeout { - return false + return false; } } sleep(Duration::from_millis(10)) @@ -495,15 +1075,101 @@ fn wait_pid_exit(pid: Pid, timeout: Option) -> bool { true } +fn try_unmount(fuse_pid: Option, mount_point: &Path) -> bool { + if let Some(fuse_pid) = fuse_pid { + sleep(Duration::from_millis(100)); + if !is_pid_exists(fuse_pid) { + return false; + } + } + if !is_mount_point(mount_point).unwrap_or(false) { + eprintln!("{:?}: not mounted!", mount_point); + return false; + } + + let mut is_busy = false; + + let res = umount(mount_point); + if res.is_ok() { + return true; + } else if let Err(err) = res { + if err == nix::Error::from(Errno::EBUSY) { + is_busy = true + } + } + + fn handle_command(cmd: &str, args: &Vec<&str>) -> (bool, bool) { + match Command::new(cmd).args(args).output() { + Ok(output) => { + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + if !stdout.is_empty() { + println!("{stdout}") + } + if !stderr.is_empty() { + eprintln!("{stderr}") + } + return ( + output.status.success(), + stderr.to_ascii_lowercase().contains("busy") + || stdout.to_ascii_lowercase().contains("busy"), + ); + } + Err(err) => { + eprintln!("Failed to execute {cmd}: {err}") + } + } + (false, false) + } + + let args = vec![mount_point.to_str().unwrap_or_default()]; + let mut fusermount_args = args.clone(); + fusermount_args.insert(0, "-u"); + + for fusermount in &["fusermount", "fusermount3"] { + if is_busy { + break; + } + let (success, busy) = handle_command(fusermount, &fusermount_args); + if success { + return true; + } else if busy { + is_busy = true + } + } + + if !is_busy { + let (success, busy) = handle_command("umount", &args); + if success { + return true; + } else if busy { + is_busy = true + } + } + + if let Some(fuse_pid) = fuse_pid { + if !is_busy && kill(fuse_pid, Signal::SIGTERM).is_ok() { + return true; + } + } + + eprintln!("Failed to unmount: {:?}", mount_point); + if fuse_pid.is_some() && is_mount_point(mount_point).unwrap_or(false) { + eprintln!("Unmount it manually!"); + } + false +} + fn wait_mount(pid: Pid, path: &PathBuf, timeout: Duration) -> bool { let start_time = Instant::now(); - spawn(move || waitpid(pid, None) ); - while !is_mount_point(path).unwrap_or(false) { + spawn(move || waitpid(pid, None)); + while !is_mounted(path).unwrap_or(false) { if !is_pid_exists(pid) { - return false + eprintln!("The mount process ended unexpectedly! PID: {pid}"); + return false; } else if start_time.elapsed() >= timeout { eprintln!("Timeout reached while waiting for mount: {:?}", path); - return false + return false; } sleep(Duration::from_millis(2)) } @@ -517,19 +1183,30 @@ fn try_setsid() { } } -fn remove_tmp_dirs(dirs: Vec<&PathBuf> ) { +fn remove_tmp_dirs(dirs: &[PathBuf], unshare_succeeded: bool) { + if let Some(dir) = dirs.first() { + if !is_mounted(dir).unwrap_or(false) { + let pidfile_ext = if !unshare_succeeded { "pid" } else { "un.pid" }; + let pid_file = dir.with_extension(pidfile_ext); + if pid_file.is_file() { + let _ = remove_file(&pid_file); + } + } + } for dir in dirs { let _ = remove_dir(dir); } } -fn create_tmp_dirs(dirs: Vec<&PathBuf>) -> Result<()> { +fn create_tmp_dirs(dirs: &[PathBuf]) -> Result<()> { if let Some(dir) = dirs.first() { create_dir_all(dir)?; for dir in dirs { if let Err(err) = set_permissions(dir, Permissions::from_mode(0o700)) { if let Some(os_error) = err.raw_os_error() { - if os_error != 30 { return Err(err) } + if os_error != 30 { + return Err(err); + } } } } @@ -540,7 +1217,7 @@ fn create_tmp_dirs(dirs: Vec<&PathBuf>) -> Result<()> { #[cfg(feature = "dwarfs")] fn get_dwfs_option(option: &str, default: &str) -> String { - let option_env = get_env_var(option); + let option_env = get_env_var!("{}", option); if option_env.is_empty() { default.into() } else { @@ -551,39 +1228,43 @@ fn get_dwfs_option(option: &str, default: &str) -> String { #[cfg(feature = "dwarfs")] fn get_dwfs_cachesize() -> String { - get_dwfs_option("DWARFS_CACHESIZE", - &if let Ok(meminfo) = ::current() { - let available_memory = meminfo.mem_available.unwrap_or(meminfo.mem_free) as f64; - let available_memory_mb = available_memory / 1024.0 / 1024.0 / 1.3; - let cache_sizes_mb: [u32; 10] = [1536, 1024, 896, 768, 640, 512, 384, 256, 128, 64]; - let cache_size_mb = cache_sizes_mb - .iter() - .find(|threshold| available_memory_mb > (**threshold as f64)).copied() - .unwrap_or(32); - format!("{}M", cache_size_mb) - } else { - DWARFS_CACHESIZE.into() - }) + get_dwfs_option( + "DWARFS_CACHESIZE", + &if let Ok(meminfo) = ::current() { + let available_memory = meminfo.mem_available.unwrap_or(meminfo.mem_free) as f64; + let available_memory_mb = available_memory / 1024.0 / 1024.0 / 1.3; + let cache_sizes_mb: [u32; 10] = [1536, 1024, 896, 768, 640, 512, 384, 256, 128, 64]; + let cache_size_mb = cache_sizes_mb + .iter() + .find(|threshold| available_memory_mb > (**threshold as f64)) + .copied() + .unwrap_or(32); + format!("{}M", cache_size_mb) + } else { + DWARFS_CACHESIZE.into() + }, + ) } #[cfg(feature = "dwarfs")] fn get_dwfs_workers(cachesize: &str, cpus: usize) -> String { - get_dwfs_option("DWARFS_WORKERS", &match cachesize { - "1536M"|"1024M" => { cpus } - "896M" => { 2 } - _ => { 1 } - }.to_string()) + get_dwfs_option( + "DWARFS_WORKERS", + &match cachesize { + "1536M" | "1024M" => cpus, + "896M" => 2, + _ => 1, + } + .to_string(), + ) } -fn mount_image(embed: &Embed, image: &Image, mount_dir: PathBuf) { - if is_mount_point(&mount_dir).unwrap_or(false) { - return +fn mount_image(embed: &Embed, image: &Image, mount_dir: PathBuf, uid: u32, gid: u32) { + if is_mounted(&mount_dir).unwrap_or(false) { + return; } - let uid = unsafe { libc::getuid() }; - let gid = unsafe { libc::getgid() }; - - let mount_dir = mount_dir.to_str().unwrap().to_string(); - let image_path = image.path.to_str().unwrap().to_string(); + let mount_dir = mount_dir.to_str().unwrap_or_default().to_string(); + let image_path = image.path.to_str().unwrap_or_default().to_string(); if image.is_dwar { #[cfg(feature = "dwarfs")] { @@ -591,32 +1272,61 @@ fn mount_image(embed: &Embed, image: &Image, mount_dir: PathBuf) { let cachesize = get_dwfs_cachesize(); let workers = get_dwfs_workers(&cachesize, cpus); let mut exec_args = vec![ - image_path, mount_dir, "-f".into(), - "-o".into(), format!("uid={uid},gid={gid}"), - "-o".into(), format!("offset={},cachesize={cachesize},workers={workers}", image.offset), - "-o".into(), "ro,nodev,tidy_strategy=time,seq_detector=1,cache_files,no_cache_image".into(), - "-o".into(), format!("blocksize={}", get_dwfs_option("DWARFS_BLOCKSIZE", DWARFS_BLOCKSIZE)), - "-o".into(), format!("readahead={}", get_dwfs_option("DWARFS_READAHEAD", DWARFS_READAHEAD)), + image_path, + mount_dir, + "-f".into(), + "-o".into(), + format!("uid={uid},gid={gid}"), + "-o".into(), + format!( + "offset={},cachesize={cachesize},workers={workers}", + image.offset + ), + "-o".into(), + "ro,nodev,tidy_strategy=time,seq_detector=1,cache_files".into(), + "-o".into(), + format!( + "blocksize={}", + get_dwfs_option("DWARFS_BLOCKSIZE", DWARFS_BLOCKSIZE) + ), + "-o".into(), + format!( + "readahead={}", + get_dwfs_option("DWARFS_READAHEAD", DWARFS_READAHEAD) + ), ]; match cachesize.as_str() { - "1536M"|"1024M" => { exec_args.append(&mut vec!["-o".into(), "clone_fd,tidy_interval=2s,tidy_max_age=10s".into()]); } - _ => { exec_args.append(&mut vec!["-o".into(), "tidy_interval=500ms,tidy_max_age=1s".into()]); } + "1536M" | "1024M" => { + exec_args.append(&mut vec![ + "-o".into(), + "clone_fd,tidy_interval=2s,tidy_max_age=10s".into(), + ]); + } + _ => { + exec_args.append(&mut vec![ + "-o".into(), + "tidy_interval=500ms,tidy_max_age=1s".into(), + ]); + } } - if get_env_var("ENABLE_FUSE_DEBUG") == "1" { + if get_env_var!("ENABLE_FUSE_DEBUG") == "1" { exec_args.append(&mut vec!["-o".into(), "debuglevel=debug".into()]); } else { exec_args.append(&mut vec!["-o".into(), "debuglevel=error".into()]); } - if get_env_var("DWARFS_PRELOAD_ALL") == "1" { + if get_env_var!("DWARFS_PRELOAD_ALL") == "1" { exec_args.append(&mut vec!["-o".into(), "preload_all".into()]); } else { exec_args.append(&mut vec!["-o".into(), "preload_category=hotness".into()]); } - let dwarfs_analysis_file = get_env_var("DWARFS_ANALYSIS_FILE"); + let dwarfs_analysis_file = get_env_var!("DWARFS_ANALYSIS_FILE"); if !dwarfs_analysis_file.is_empty() { - exec_args.append(&mut vec!["-o".into(), format!("analysis_file={dwarfs_analysis_file}")]); + exec_args.append(&mut vec![ + "-o".into(), + format!("analysis_file={dwarfs_analysis_file}"), + ]); } - if get_env_var("DWARFS_USE_MMAP") == "1" { + if get_env_var!("DWARFS_USE_MMAP") == "1" { exec_args.append(&mut vec!["-o".into(), "block_allocator=mmap".into()]); } else { exec_args.append(&mut vec!["-o".into(), "block_allocator=malloc".into()]); @@ -627,12 +1337,17 @@ fn mount_image(embed: &Embed, image: &Image, mount_dir: PathBuf) { #[cfg(feature = "squashfs")] { let mut exec_args = vec![ - image_path, mount_dir, "-f".into(), - "-o".into(), "ro,nodev".into(), - "-o".into(), format!("uid={uid},gid={gid}"), - "-o".into(), format!("offset={}", image.offset) + image_path, + mount_dir, + "-f".into(), + "-o".into(), + "ro,nodev".into(), + "-o".into(), + format!("uid={uid},gid={gid}"), + "-o".into(), + format!("offset={}", image.offset), ]; - if get_env_var("ENABLE_FUSE_DEBUG") == "1" { + if get_env_var!("ENABLE_FUSE_DEBUG") == "1" { exec_args.append(&mut vec!["-o".into(), "debug".into()]); } embed.squashfuse(exec_args) @@ -640,11 +1355,17 @@ fn mount_image(embed: &Embed, image: &Image, mount_dir: PathBuf) { } } -fn extract_image(embed: &Embed, image: &Image, mut extract_dir: PathBuf, is_extract_run: bool, pattern: Option<&String>) { +fn extract_image( + embed: &Embed, + image: &Image, + mut extract_dir: PathBuf, + is_extract_run: bool, + pattern: Option<&String>, +) { if is_extract_run { if let Ok(dir) = extract_dir.read_dir() { - if dir.flatten().any(|entry|entry.path().exists()) { - return + if dir.flatten().any(|entry| entry.path().exists()) { + return; } } } @@ -661,7 +1382,7 @@ fn extract_image(embed: &Embed, image: &Image, mut extract_dir: PathBuf, is_extr } } } - let extract_dir = extract_dir.to_str().unwrap().to_string(); + let extract_dir = extract_dir.to_str().unwrap_or_default().to_string(); if let Err(err) = create_dir_all(&extract_dir) { eprintln!("Failed to create extract dir: {err}: {extract_dir}"); exit(1) @@ -671,24 +1392,28 @@ fn extract_image(embed: &Embed, image: &Image, mut extract_dir: PathBuf, is_extr if !is_extract_run { let _ = remove_file(&applink_dir); if let Err(err) = symlink(&extract_dir, &applink_dir) { - eprintln!("Failed to create squashfs-root symlink to extract dir: {err}"); - exit(1) + eprintln!("Warning: failed to create squashfs-root symlink to extract dir: {err}"); } } } - let image_path = image.path.to_str().unwrap().to_string(); + let image_path = image.path.to_str().unwrap_or_default().to_string(); if image.is_dwar { #[cfg(feature = "dwarfs")] { let cachesize = get_dwfs_cachesize(); let mut exec_args = vec![ - "--input".into(), image_path, + "--input".into(), + image_path, "--log-level=error".into(), format!("--cache-size={cachesize}"), format!("--image-offset={}", image.offset), - format!("--num-workers={}", get_dwfs_workers(&cachesize, num_cpus::get())), - "--output".into(), extract_dir, - "--stdout-progress".into() + format!( + "--num-workers={}", + get_dwfs_workers(&cachesize, num_cpus::get()) + ), + "--output".into(), + extract_dir, + "--stdout-progress".into(), ]; if let Some(pattern) = pattern { exec_args.append(&mut vec!["--pattern".into(), pattern.to_string()]); @@ -698,10 +1423,13 @@ fn extract_image(embed: &Embed, image: &Image, mut extract_dir: PathBuf, is_extr } else { #[cfg(feature = "squashfs")] { - let mut exec_args = vec!["-f".into(), - "-d".into(), extract_dir, - "-o".into(), image.offset.to_string(), - image_path + let mut exec_args = vec![ + "-f".into(), + "-d".into(), + extract_dir, + "-o".into(), + image.offset.to_string(), + image_path, ]; if let Some(pattern) = pattern { exec_args.push(pattern.into()) @@ -711,29 +1439,27 @@ fn extract_image(embed: &Embed, image: &Image, mut extract_dir: PathBuf, is_extr } } -fn try_set_portable(kind: &str, dir: &PathBuf) { +fn try_set_portable_dir(dir: &PathBuf, env_var: &str, default_path: Option<&str>) { + let real_env_var = format!("REAL_{}", env_var); if dir.is_dir() { - match kind { - "home" => { - eprintln!("Setting $HOME to {:?}", dir); - env::set_var("HOME", dir) - } - "config" => { - eprintln!("Setting $XDG_CONFIG_HOME to {:?}", dir); - env::set_var("XDG_CONFIG_HOME", dir) - } - "share" => { - eprintln!("Setting $XDG_DATA_HOME to {:?}", dir); - env::set_var("XDG_DATA_HOME", dir) + if get_env_var!("{}", real_env_var).is_empty() { + if let Ok(current_value) = env::var(env_var) { + env::set_var(&real_env_var, current_value); + } else if let Some(default) = default_path { + if let Ok(home) = env::var("HOME") { + let default_dir = PathBuf::from(home).join(default); + env::set_var(&real_env_var, default_dir); + } } - _ => {} } + eprintln!("Setting ${} to {:?}", env_var, dir); + env::set_var(env_var, dir); } } fn parse_reuse_check_delay(delay: &str) -> Option { if delay == "inf" { - return None + return None; } let default_delay = Some(Duration::from_secs(1)); let mut chars = delay.chars(); @@ -753,12 +1479,14 @@ fn parse_reuse_check_delay(delay: &str) -> Option { "h" => 3600, _ => return default_delay, }; - return Some(Duration::from_secs(num * multiplier)) + return Some(Duration::from_secs(num * multiplier)); } } if !num_part.is_empty() { num_part.parse().ok().map(Duration::from_secs) - } else { default_delay } + } else { + default_delay + } } fn try_read_dotenv(dotenv_path: &PathBuf, dotenv_string: &str) { @@ -787,17 +1515,36 @@ fn try_read_dotenv(dotenv_path: &PathBuf, dotenv_string: &str) { } } -fn signals_handler(pid: Pid, selfexit: bool) { - let mut signals = Signals::new([SIGINT, SIGTERM, SIGQUIT]).unwrap(); - let _ = signals.handle(); +fn signals_handler(pid: Pid, mount_point: &Path, killpid: bool, selfexit: bool) { + let sig_list = [SIGINT, SIGTERM, SIGQUIT, SIGHUP, SIGUSR1, SIGUSR2]; + let mut signals = match Signals::new(sig_list) { + Ok(sig) => sig, + Err(err) => { + eprintln!("Failed to register signal handlers: {err}"); + return; + } + }; + let _handle = signals.handle(); for signal in signals.forever() { - match signal { - SIGINT | SIGTERM | SIGQUIT | SIGHUP => { - let _ = kill(pid, Signal::SIGTERM); - if selfexit { exit(0) }; - break + if sig_list.contains(&signal) { + if killpid { + if let Ok(signal_enum) = Signal::try_from(signal) { + let _ = kill(pid, signal_enum); + } + } else { + try_unmount(Some(pid), mount_point); + unsafe { + for &sig in sig_list.iter() { + libc::signal(sig, libc::SIG_DFL); + } + } + } + if selfexit { + exit(0) + }; + if !killpid { + break; } - _ => {} } } } @@ -808,9 +1555,9 @@ fn hash_string(data: &str) -> String { hasher.finish().to_string() } -fn fast_hash_file(path: &PathBuf, offset: u64) -> Result { +fn fast_hash_file(path: &Path, offset: u64) -> Result { let mut file = File::open(path)?; - let file_size = get_file_size(path)?.saturating_sub(offset); + let file_size = file.metadata()?.len().saturating_sub(offset); let mut buffer = [0u8; 48]; file.seek(SeekFrom::Start(offset))?; file.read_exact(&mut buffer[0..16])?; @@ -821,19 +1568,33 @@ fn fast_hash_file(path: &PathBuf, offset: u64) -> Result { Ok(xxh3_64(&buffer) as u32) } -fn print_usage(portable_home: &PathBuf, portable_share: &PathBuf, portable_config: &PathBuf, self_exe_dotenv: &PathBuf) { +fn print_usage( + portable_home: &PathBuf, + portable_share: &PathBuf, + portable_config: &PathBuf, + portable_cache: &PathBuf, + self_exe_dotenv: &PathBuf, +) { println!("{} v{URUNTIME_VERSION} Repository: {} Runtime options: --{ARG_PFX}-extract [PATTERN] Extract content from embedded filesystem image If pattern is passed, only extract matching files - --{ARG_PFX}-extract-and-run [ARGS] Run the {SELF_NAME} afer extraction without using FUSE + --{ARG_PFX}-extract-and-run [ARGS] Run the {SELF_NAME} after extraction without using FUSE --{ARG_PFX}-offset Print byte offset to start of embedded filesystem image --{ARG_PFX}-portable-home Create a portable home folder to use as $HOME --{ARG_PFX}-portable-share Create a portable share folder to use as $XDG_DATA_HOME --{ARG_PFX}-portable-config Create a portable config folder to use as $XDG_CONFIG_HOME + --{ARG_PFX}-portable-cache Create a portable cache folder to use as $XDG_CACHE_HOME --{ARG_PFX}-help Print this help + --{ARG_PFX}-unshare Try to use unshare user and mount namespaces + --{ARG_PFX}-unshare-root Use unshare and map the current user to UID/GID 0 + --{ARG_PFX}-unshare-uid UID Use unshare and map the current UID to UID + --{ARG_PFX}-unshare-gid GID Use unshare and map the current GID to GID + --{ARG_PFX}-unshare-drop-caps Use unshare and drop capabilities before the application + --{ARG_PFX}-unshare-fallback-drop-caps + Drop capabilities only when unshare is selected as fallback --{ARG_PFX}-version Print version of Runtime --{ARG_PFX}-signature Print digital signature embedded in {SELF_NAME} --{ARG_PFX}-addsign 'SIGN|/file' Add digital signature to {SELF_NAME} @@ -864,11 +1625,14 @@ fn print_usage(portable_home: &PathBuf, portable_share: &PathBuf, portable_confi println!(" --{ARG_PFX}-mkdwarfs [ARGS] Launch mkdwarfs"); #[cfg(feature = "dwarfs")] println!(" --{ARG_PFX}-dwarfsextract [ARGS] Launch dwarfsextract"); - println!(" + println!( + " Also you can create a hardlink, symlink or rename the runtime with - the name of the built-in utility to use it directly."); + the name of the built-in utility to use it directly." + ); - println!("\n Portable home and config: + println!( + "\n Portable home and config: If you would like the application contained inside this {SELF_NAME} to store its data alongside this {SELF_NAME} rather than in your home directory, then you can @@ -883,33 +1647,44 @@ fn print_usage(portable_home: &PathBuf, portable_share: &PathBuf, portable_confi for portable-config: {:?} + for portable-cache: + {:?} + Or you can invoke this {SELF_NAME} with the --{ARG_PFX}-portable-home or - --{ARG_PFX}-portable-share or --{ARG_PFX}-portable-config option, - which will create this directory for you. + --{ARG_PFX}-portable-share or --{ARG_PFX}-portable-config or + --{ARG_PFX}-portable-cache option, which will create this directory for you. As long as the directory exists and is neither moved nor renamed, the application contained inside this {SELF_NAME} to store its data in this - directory rather than in your home directory", portable_home, portable_share, portable_config); + directory rather than in your home directory", + portable_home, portable_share, portable_config, portable_cache + ); println!("\n Environment variables: URUNTIME Path to uruntime URUNTIME_DIR Path to uruntime directory - {}_EXTRACT_AND_RUN=1 Run the {SELF_NAME} afer extraction without using FUSE + {ENV_NAME}_UNSHARE=1 Try to use unshare user and mount namespaces + {ENV_NAME}_UNSHARE=2 Use unshare and drop capabilities before the application + {ENV_NAME}_UNSHARE=3 Drop capabilities only when unshare is selected as fallback + {ENV_NAME}_UNSHARE_ROOT=1 Map to root (UID 0, GID 0) in user namespace + {ENV_NAME}_UNSHARE_UID=0 Map to specified UID in user namespace + {ENV_NAME}_UNSHARE_GID=0 Map to specified GID in user namespace + + {ENV_NAME}_EXTRACT_AND_RUN=1 Run the {SELF_NAME} after extraction without using FUSE NO_CLEANUP=1 Do not clear the unpacking directory after closing when using extract and run option for reuse extracted data NO_UNMOUNT=1 Do not unmount the mount directory after closing for reuse mount point TMPDIR=/path Specifies a custom path for mounting or extracting the image - URUNTIME_TARGET_DIR=/path Specifies the exact path for mounting or extracting the image - REUSE_CHECK_DELAY=5s Specifies the delay between checks of using the image dir (inf|1|1s|1m|1h) + {ENV_NAME}_TARGET_DIR=/path Specifies the exact path for mounting or extracting the image + REUSE_CHECK_DELAY=5s Specifies the delay between checks of using the image dir (0|inf|1|1s|1m|1h) FUSERMOUNT_PROG=/path Specifies a custom path for fusermount ENABLE_FUSE_DEBUG=1 Enables debug mode for the mounted filesystem - TARGET_{}=/path Operate on a target {SELF_NAME} rather than this file itself - NO_MEMFDEXEC=1 Do not use memfd-exec (use a temporary file instead)", - ARG_PFX.to_uppercase(), SELF_NAME.to_uppercase()); + TARGET_{ENV_NAME}=/path Operate on a target {SELF_NAME} rather than this file itself + NO_MEMFDEXEC=1 Do not use memfd-exec (use a temporary file instead)"); #[cfg(feature = "dwarfs")] { - println!(" DWARFS_WORKERS=2 Number of worker threads for DwarFS (default: equal CPU threads) + println!(" DWARFS_WORKERS=2 Number of worker threads for DwarFS (default: equal CPU threads) DWARFS_CACHESIZE=1024M Size of the block cache, in bytes for DwarFS (suffixes K, M, G) DWARFS_BLOCKSIZE=512K Size of the block file I/O, in bytes for DwarFS (suffixes K, M, G) DWARFS_READAHEAD=32M Set readahead size, in bytes for DwarFS (suffixes K, M, G) @@ -928,107 +1703,189 @@ fn print_usage(portable_home: &PathBuf, portable_share: &PathBuf, portable_confi fn main() { let embed = Embed::new(); - let mut exec_args: Vec = env::args().collect(); - let arg0 = &exec_args.remove(0); + let mut args = env::args(); + let arg0 = args.next().unwrap_or_default(); + let mut exec_args: Vec = args.collect(); + let arg0_name = basename(&arg0); - match basename(arg0).as_str() { + match arg0_name { #[cfg(feature = "squashfs")] - "squashfuse" => { embed.squashfuse(exec_args); return } + "squashfuse" => { + embed.squashfuse(exec_args); + return; + } #[cfg(feature = "squashfs")] - "unsquashfs" => { embed.unsquashfs(exec_args); return } + "unsquashfs" => { + embed.unsquashfs(exec_args); + return; + } #[cfg(feature = "squashfs")] - "sqfscat" => { embed.sqfscat(exec_args); return } + "sqfscat" => { + embed.sqfscat(exec_args); + return; + } #[cfg(all(not(feature = "lite"), feature = "squashfs"))] - "mksquashfs" => { embed.mksquashfs(exec_args); return } + "mksquashfs" => { + embed.mksquashfs(exec_args); + return; + } #[cfg(all(not(feature = "lite"), feature = "squashfs"))] - "sqfstar" => { embed.sqfstar(exec_args); return } + "sqfstar" => { + embed.sqfstar(exec_args); + return; + } #[cfg(feature = "dwarfs")] - "dwarfs" => { embed.dwarfs(exec_args); return } + "dwarfs" => { + embed.dwarfs(exec_args); + return; + } #[cfg(all(not(feature = "lite"), feature = "dwarfs"))] - "dwarfsck" => { embed.dwarfsck(exec_args); return } + "dwarfsck" => { + embed.dwarfsck(exec_args); + return; + } #[cfg(all(not(feature = "lite"), feature = "dwarfs"))] - "mkdwarfs" => { embed.mkdwarfs(exec_args); return } + "mkdwarfs" => { + embed.mkdwarfs(exec_args); + return; + } #[cfg(feature = "dwarfs")] - "dwarfsextract" => { embed.dwarfsextract(exec_args); return } + "dwarfsextract" => { + embed.dwarfsextract(exec_args); + return; + } + "fusermount" | "fusermount3" => { + let mut umount = false; + let mut mount_point = String::new(); + for arg in &exec_args { + if arg == "-u" || arg == "--unmount" { + umount = true + } else if !arg.starts_with('-') { + mount_point = arg.clone(); + break; + } + } + let current_path = env::var("PATH").unwrap_or_default(); + let filtered_path = current_path + .split(':') + .filter(|path| !path.starts_with("/tmp/.path")) + .collect::>() + .join(":"); + env::set_var("PATH", filtered_path); + drop(current_path); + if umount && !mount_point.is_empty() { + if !try_unmount(None, Path::new(&mount_point)) { + exit(1) + } + return; + } + let err = Command::new(arg0_name).args(&exec_args).exec(); + eprintln!("Failed to execute {arg0_name}: {err}"); + exit(1) + } _ => {} } - let arg1 = if !exec_args.is_empty() { - exec_args[0].to_string() - } else {"".into()}; + let unshare_cli = parse_unshare_cli_options(&mut exec_args, ARG_PFX).unwrap_or_else(|error| { + eprintln!("Invalid unshare option: {error}"); + exit(2) + }); + let arg1 = exec_args.first().map(String::as_str).unwrap_or_default(); if !arg1.is_empty() { match arg1 { - arg if arg == format!("--{ARG_PFX}-version") => { + arg if is_runtime_option(arg, "version") => { println!("v{URUNTIME_VERSION}"); - return + return; } #[cfg(feature = "squashfs")] - arg if arg == format!("--{ARG_PFX}-squashfuse") => { + arg if is_runtime_option(arg, "squashfuse") => { embed.squashfuse(exec_args[1..].to_vec()); - return + return; } #[cfg(feature = "squashfs")] - arg if arg == format!("--{ARG_PFX}-unsquashfs") => { + arg if is_runtime_option(arg, "unsquashfs") => { embed.unsquashfs(exec_args[1..].to_vec()); - return + return; } #[cfg(feature = "squashfs")] - arg if arg == format!("--{ARG_PFX}-sqfscat") => { + arg if is_runtime_option(arg, "sqfscat") => { embed.sqfscat(exec_args[1..].to_vec()); - return + return; } #[cfg(all(not(feature = "lite"), feature = "squashfs"))] - arg if arg == format!("--{ARG_PFX}-mksquashfs") => { + arg if is_runtime_option(arg, "mksquashfs") => { embed.mksquashfs(exec_args[1..].to_vec()); - return + return; } #[cfg(all(not(feature = "lite"), feature = "squashfs"))] - arg if arg == format!("--{ARG_PFX}-sqfstar") => { + arg if is_runtime_option(arg, "sqfstar") => { embed.sqfstar(exec_args[1..].to_vec()); - return + return; } #[cfg(feature = "dwarfs")] - arg if arg == format!("--{ARG_PFX}-dwarfs") => { + arg if is_runtime_option(arg, "dwarfs") => { embed.dwarfs(exec_args[1..].to_vec()); - return + return; } #[cfg(all(not(feature = "lite"), feature = "dwarfs"))] - arg if arg == format!("--{ARG_PFX}-dwarfsck") => { + arg if is_runtime_option(arg, "dwarfsck") => { embed.dwarfsck(exec_args[1..].to_vec()); - return + return; } #[cfg(all(not(feature = "lite"), feature = "dwarfs"))] - arg if arg == format!("--{ARG_PFX}-mkdwarfs") => { + arg if is_runtime_option(arg, "mkdwarfs") => { embed.mkdwarfs(exec_args[1..].to_vec()); - return + return; } #[cfg(feature = "dwarfs")] - arg if arg == format!("--{ARG_PFX}-dwarfsextract") => { + arg if is_runtime_option(arg, "dwarfsextract") => { embed.dwarfsextract(exec_args[1..].to_vec()); - return + return; } _ => {} } } - let uruntime = ¤t_exe().unwrap(); - let target_image = &PathBuf::from(get_env_var(&format!("TARGET_{}", SELF_NAME.to_uppercase()))); - let self_exe = if target_image.is_file() { target_image } else { uruntime }; + let uruntime = ¤t_exe().unwrap_or_else(|err| { + eprintln!("Failed to get self runtime exe path: {err}"); + exit(1) + }); + let target_image_value = get_env_var!("TARGET_{}", ENV_NAME); + let target_image = &PathBuf::from(&target_image_value); + let self_exe = if !target_image_value.is_empty() && target_image.is_file() { + target_image + } else { + uruntime + }; - let runtime = get_runtime(self_exe).unwrap_or_else(|err|{ + let runtime = get_runtime(self_exe).unwrap_or_else(|err| { eprintln!("Failed to get runtime: {err}"); exit(1) }); let runtime_size = runtime.size; - let uruntime_dir = uruntime.parent().unwrap(); - let self_exe_dir = self_exe.parent().unwrap(); - let self_exe_name = self_exe.file_name().unwrap().to_str().unwrap(); + let uruntime_dir = uruntime.parent().unwrap_or_else(|| { + eprintln!("Failed to get self runtime parent dir!"); + exit(1) + }); + let self_exe_dir = self_exe.parent().unwrap_or_else(|| { + eprintln!("Failed to get runtime parent dir!"); + exit(1) + }); + let self_exe_name = self_exe + .file_name() + .unwrap_or_else(|| { + eprintln!("Failed to get runtime name!"); + exit(1) + }) + .to_str() + .unwrap_or_default(); let portable_home = &self_exe_dir.join(format!("{self_exe_name}.home")); let portable_share = &self_exe_dir.join(format!("{self_exe_name}.share")); let portable_config = &self_exe_dir.join(format!("{self_exe_name}.config")); + let portable_cache = &self_exe_dir.join(format!("{self_exe_name}.cache")); env::set_var("URUNTIME", uruntime); env::set_var("URUNTIME_DIR", uruntime_dir); @@ -1038,127 +1895,179 @@ fn main() { let mut is_mount_only = false; let mut is_extract_run = false; - let mut is_noclenup = !matches!(URUNTIME_CLEANUP.replace("URUNTIME_CLEANUP=", "=").as_str(), "=1"); - - if get_env_var(&format!("{}_EXTRACT_AND_RUN", ARG_PFX.to_uppercase())) == "1" { + let mut is_noclenup = URUNTIME_CLEANUP.strip_prefix("URUNTIME_CLEANUP") != Some("=1"); + let unshare_mode = URUNTIME_UNSHARE + .strip_prefix("URUNTIME_UNSHARE") + .unwrap_or_default(); + let (mut is_unshare, mut drop_caps, mut drop_caps_on_fallback) = + embedded_unshare_policy(unshare_mode); + is_unshare |= unshare_cli.enable; + drop_caps |= unshare_cli.drop_caps; + drop_caps_on_fallback |= unshare_cli.drop_caps_on_fallback; + if get_env_var!("{}_EXTRACT_AND_RUN", ENV_NAME) == "1" { is_extract_run = true } + let arg1 = exec_args.first().map(String::as_str).unwrap_or_default(); + let extract_and_run = is_runtime_option(arg1, "extract-and-run"); + let explicit_unshare = is_runtime_option(arg1, "unshare"); if !arg1.is_empty() { match arg1 { - arg if arg == format!("--{ARG_PFX}-help") => { - print_usage(portable_home, portable_share, portable_config, self_exe_dotenv); - return + arg if is_runtime_option(arg, "help") => { + print_usage( + portable_home, + portable_share, + portable_config, + portable_cache, + self_exe_dotenv, + ); + return; } - arg if arg == format!("--{ARG_PFX}-portable-home") => { + arg if is_runtime_option(arg, "portable-home") => { if let Err(err) = create_dir(portable_home) { - eprintln!("Failed to create portable home directory: {:?}: {err}", portable_home) + eprintln!( + "Failed to create portable home directory: {:?}: {err}", + portable_home + ) } println!("Portable home directory created: {:?}", portable_home); - return + return; } - arg if arg == format!("--{ARG_PFX}-portable-share") => { + arg if is_runtime_option(arg, "portable-share") => { if let Err(err) = create_dir(portable_share) { - eprintln!("Failed to create portable share directory: {:?}: {err}", portable_share) + eprintln!( + "Failed to create portable share directory: {:?}: {err}", + portable_share + ) } println!("Portable share directory created: {:?}", portable_share); - return + return; } - arg if arg == format!("--{ARG_PFX}-portable-config") => { + arg if is_runtime_option(arg, "portable-config") => { if let Err(err) = create_dir(portable_config) { - eprintln!("Failed to create portable config directory: {:?}: {err}", portable_config) + eprintln!( + "Failed to create portable config directory: {:?}: {err}", + portable_config + ) } println!("Portable config directory created: {:?}", portable_config); - return + return; + } + arg if is_runtime_option(arg, "portable-cache") => { + if let Err(err) = create_dir(portable_cache) { + eprintln!( + "Failed to create portable cache directory: {:?}: {err}", + portable_cache + ) + } + println!("Portable cache directory created: {:?}", portable_cache); + return; } - arg if arg == format!("--{ARG_PFX}-offset") => { + arg if is_runtime_option(arg, "offset") => { println!("{runtime_size}"); - return + return; } - arg if arg == format!("--{ARG_PFX}-updateinfo") || - arg == format!("--{ARG_PFX}-updateinformation") => { + arg if is_runtime_option(arg, "updateinfo") + || is_runtime_option(arg, "updateinformation") => + { let updateinfo = get_section_data(&runtime.headers_bytes, ".upd_info") - .unwrap_or_else(|err|{ + .unwrap_or_else(|err| { eprintln!("Failed to get update info: {err}"); exit(1) - }); + }); println!("{updateinfo}"); - return + return; } - arg if arg == format!("--{ARG_PFX}-addupdinfo") => { + arg if is_runtime_option(arg, "addupdinfo") => { if let Err(err) = add_section_data(&runtime, ".upd_info", &exec_args) { eprintln!("Failed to add update info: {err}"); exit(1) }; - return + return; } - arg if arg == format!("--{ARG_PFX}-signature") => { + arg if is_runtime_option(arg, "signature") => { let signature = get_section_data(&runtime.headers_bytes, ".sha256_sig") - .unwrap_or_else(|err|{ + .unwrap_or_else(|err| { eprintln!("Failed to get signature info: {err}"); exit(1) - }); + }); println!("{signature}"); - return + return; } - arg if arg == format!("--{ARG_PFX}-addsign") => { + arg if is_runtime_option(arg, "addsign") => { if let Err(err) = add_section_data(&runtime, ".sha256_sig", &exec_args) { eprintln!("Failed to add signature info: {err}"); exit(1) }; - return + return; } - arg if arg == format!("--{ARG_PFX}-envs") => { + arg if is_runtime_option(arg, "envs") => { println!("{}", runtime.envs); - return + return; } - arg if arg == format!("--{ARG_PFX}-addenvs") => { + arg if is_runtime_option(arg, "addenvs") => { if let Err(err) = add_section_data(&runtime, ".envs", &exec_args) { eprintln!("Failed to add envs: {err}"); exit(1) }; - return - } - ref arg if arg == &format!("--{ARG_PFX}-extract-and-run") => { - exec_args.remove(0); - is_extract_run = true + return; } _ => {} } } + if extract_and_run { + exec_args.remove(0); + is_extract_run = true; + } else if explicit_unshare { + exec_args.remove(0); + is_unshare = true; + } let image = get_image(self_exe, runtime_size).unwrap_or_else(|err|{ eprintln!("Failed to get image: {err}"); + eprintln!("The embedded filesystem image may be corrupted, truncated by 'strip', or not yet included in this executable"); exit(1) }); + let arg1 = exec_args.first().map(String::as_str).unwrap_or_default(); if !arg1.is_empty() { match arg1 { - arg if arg == format!("--{ARG_PFX}-extract") => { - extract_image(&embed, &image, PathBuf::from("."), - false, exec_args.get(1)); - return - } - arg if arg == format!("--{ARG_PFX}-mount") => { - is_mount_only = true + arg if is_runtime_option(arg, "extract") => { + extract_image(&embed, &image, PathBuf::from("."), false, exec_args.get(1)); + return; } + arg if is_runtime_option(arg, "mount") => is_mount_only = true, _ => {} } } - let uruntime_extract = - match URUNTIME_EXTRACT.replace("URUNTIME_EXTRACT=", "=").as_str() { - "=1" => { is_extract_run = true; 1 } - "=2" => { 2 } - "=3" => { 3 } - _ => { 0 } + let uruntime_extract = match URUNTIME_EXTRACT + .strip_prefix("URUNTIME_EXTRACT") + .unwrap_or_default() + { + "=1" => { + is_extract_run = true; + 1 + } + "=2" => 2, + "=3" => 3, + _ => 0, }; - let mut reuse_check_delay = get_env_var("REUSE_CHECK_DELAY"); + let mut reuse_check_delay = get_env_var!("REUSE_CHECK_DELAY"); - let (mut is_remp_mount, default_delay) = - match URUNTIME_MOUNT.replace("URUNTIME_MOUNT=", "=").as_str() { - "=0" => (true, if is_extract_run { Some(REUSE_CHECK_DELAY) } else { Some("inf") }), + let (mut is_remp_mount, default_delay) = match URUNTIME_MOUNT + .strip_prefix("URUNTIME_MOUNT") + .unwrap_or_default() + { + "=0" => ( + true, + if is_extract_run { + Some(REUSE_CHECK_DELAY) + } else { + Some("inf") + }, + ), "=1" => (false, None), "=2" => (true, Some("30m")), "=3" => (true, Some(REUSE_CHECK_DELAY)), @@ -1168,31 +2077,38 @@ fn main() { if let Some(default) = default_delay { if reuse_check_delay.is_empty() { reuse_check_delay = default.into(); + } else if reuse_check_delay == "0" { + is_remp_mount = false } }; - let target_dir = get_env_var("URUNTIME_TARGET_DIR"); - let mut tmp_dir: PathBuf; - let tmp_dirs: Vec<&PathBuf>; - #[cfg(not(feature = "appimage"))] - let ruid_dir: PathBuf; - #[cfg(not(feature = "appimage"))] - let mnt_dir: PathBuf; + let target_dir = get_env_var!("{}_TARGET_DIR", ENV_NAME); + let target_dir_is_empty = target_dir.is_empty(); + + let uid: u32 = unsafe { libc::getuid() }; + let gid = unsafe { libc::getgid() }; - if target_dir.is_empty() { - tmp_dir = env::temp_dir(); + let (tmp_dir, tmp_dirs) = if target_dir_is_empty { + let base_tmp_dir = env::temp_dir(); let mut self_hash = "".to_string(); - let first5name: String = self_exe_name.split(".").next() - .unwrap_or(self_exe_name).chars().take(5).collect(); + let first5name: String = self_exe_name + .split(".") + .next() + .unwrap_or(self_exe_name) + .chars() + .filter(|c| c.is_ascii_alphanumeric()) + .take(5) + .collect(); if is_extract_run || is_remp_mount { - let uid = unsafe { libc::getuid() }; - self_hash = hash_string(&( - xxh3_64(&runtime.headers_bytes) as u32 + - fast_hash_file(&image.path, image.offset).unwrap_or_else(|err|{ - eprintln!("Failed to get image hash: {err}"); - exit(1)}) + - uid - ).to_string()) + self_hash = hash_string( + &(xxh3_64(&runtime.headers_bytes) as u32 + + fast_hash_file(&image.path, image.offset).unwrap_or_else(|err| { + eprintln!("Failed to get image hash: {err}"); + exit(1) + }) + + uid) + .to_string(), + ) } cfg_if! { @@ -1204,12 +2120,11 @@ fn main() { } else { format!(".mount_{first5name}{}", random_string(6)) }; - tmp_dir = tmp_dir.join(tmp_dir_name); - tmp_dirs = vec![&tmp_dir]; + let tmp_dir = base_tmp_dir.join(tmp_dir_name); + (tmp_dir.clone(), vec![tmp_dir]) } else { - let uid = unsafe { libc::getuid() }; - ruid_dir = tmp_dir.join(format!(".r{uid}")); - mnt_dir = ruid_dir.join("mnt"); + let ruid_dir = base_tmp_dir.join(format!(".r{uid}")); + let mnt_dir = ruid_dir.join("mnt"); let tmp_dir_name: String = if is_extract_run && !is_mount_only { format!("{first5name}extr{self_hash}") } else if is_remp_mount { @@ -1217,170 +2132,297 @@ fn main() { } else { format!("{first5name}{}", random_string(6)) }; - tmp_dir = mnt_dir.join(tmp_dir_name); - tmp_dirs = vec![&tmp_dir, &mnt_dir, &ruid_dir]; + let tmp_dir = mnt_dir.join(tmp_dir_name); + (tmp_dir.clone(), vec![tmp_dir, mnt_dir, ruid_dir]) } } - drop(first5name); } else { - env::remove_var("URUNTIME_TARGET_DIR"); - tmp_dir = PathBuf::from(target_dir); - tmp_dirs = vec![&tmp_dir] - } - + env::remove_var(format!("{ENV_NAME}_TARGET_DIR")); + let tmp_dir = PathBuf::from(target_dir); + (tmp_dir.clone(), vec![tmp_dir]) + }; drop(runtime); - if (!is_extract_run || is_mount_only) && !check_fuse() { - check_extract!(is_mount_only, uruntime_extract, self_exe, { - is_extract_run = true - }); + let mut unshare_succeeded = false; + let env_unshare = get_env_var!("{}_UNSHARE", ENV_NAME); + let (env_enables_unshare, env_drops_caps, env_drops_caps_on_fallback) = + environment_drop_caps_policy(&env_unshare); + is_unshare |= env_enables_unshare; + drop_caps |= env_drops_caps; + drop_caps_on_fallback |= env_drops_caps_on_fallback; + + let env_unshare_root = get_env_var!("{}_UNSHARE_ROOT", ENV_NAME) == "1"; + let (unshare_uid, unshare_gid) = if unshare_cli.root || env_unshare_root { + ("0".into(), "0".into()) + } else { + ( + unshare_cli + .uid + .unwrap_or_else(|| get_env_var!("{}_UNSHARE_UID", ENV_NAME)), + unshare_cli + .gid + .unwrap_or_else(|| get_env_var!("{}_UNSHARE_GID", ENV_NAME)), + ) + }; + if !unshare_uid.is_empty() || !unshare_gid.is_empty() || env_enables_unshare { + is_unshare = true + } + if is_unshare && drop_caps_on_fallback { + drop_caps = true; } - if is_mount_only { - println!("{}", tmp_dir.display()); - is_extract_run = false - } else if is_extract_run { - is_noclenup = get_env_var("NO_CLEANUP") == "1" + let mut is_tmpdir_exists = false; + let mut is_unshare_remp = false; + let mut child_pid = Pid::from_raw(0); + if is_remp_mount && !is_extract_run { + if let Some(pid) = try_reuse_unshare_mount_point(&tmp_dir) { + is_tmpdir_exists = true; + unshare_succeeded = true; + is_unshare_remp = true; + child_pid = pid + } + } + if fallback_should_drop_capabilities(unshare_succeeded && !is_unshare, drop_caps_on_fallback) { + drop_caps = true; } - if !is_extract_run && get_env_var("NO_UNMOUNT") == "1" { - is_remp_mount = true; - reuse_check_delay = "inf".into() + if !is_unshare_remp { + is_tmpdir_exists = is_mounted(&tmp_dir).unwrap_or(false) + || if let Ok(dir) = tmp_dir.read_dir() { + dir.flatten().any(|entry| entry.path().exists()) + } else { + false + }; } - let is_tmpdir_exists = is_mount_point(&tmp_dir).unwrap_or(false) || - if let Ok(dir) = tmp_dir.read_dir() { - dir.flatten().any(|entry|entry.path().exists()) - } else { false }; + if is_remp_mount && !is_extract_run && !is_unshare_remp { + child_pid = read_mount_pid_file(&tmp_dir, "pid").unwrap_or(Pid::from_raw(0)) + } - match unsafe { fork() } { - Ok(ForkResult::Parent { child: child_pid }) => { - if !is_tmpdir_exists { - if is_extract_run { - if let Err(err) = waitpid(child_pid, None) { - eprintln!("Failed to extract image: {err}"); - remove_tmp_dirs(tmp_dirs); - exit(1) - } - } else if !wait_mount(child_pid, &tmp_dir, Duration::from_secs(1)) { - remove_tmp_dirs(tmp_dirs); - check_extract!(is_mount_only, uruntime_extract, self_exe, { - let err = Command::new(self_exe) - .env(format!("{}_EXTRACT_AND_RUN", ARG_PFX.to_uppercase()), "1") - .args(&exec_args) - .exec(); - eprintln!("Failed to exec: {:?}: {err}", self_exe); - }); + if !is_tmpdir_exists { + if !is_unshare_remp && is_unshare { + unshare_succeeded = try_unshare(uid, gid, &unshare_uid, &unshare_gid); + } + + if !is_extract_run || is_mount_only { + let unshare_was_requested = is_unshare; + let fuse_available = check_fuse( + uruntime, + uid, + gid, + &unshare_uid, + &unshare_gid, + &mut unshare_succeeded, + &mut is_unshare, + ); + if fallback_should_drop_capabilities( + unshare_succeeded && !unshare_was_requested, + drop_caps_on_fallback, + ) { + drop_caps = true; + } + if !fuse_available { + check_extract!(is_mount_only, uruntime_extract, self_exe, { + is_extract_run = true + }); + } + } + drop(unshare_uid); + drop(unshare_gid); + + if is_mount_only { + is_extract_run = false + } else if is_extract_run { + is_noclenup = get_env_var!("NO_CLEANUP") == "1" + } + + if !is_extract_run && get_env_var!("NO_UNMOUNT") == "1" { + is_remp_mount = true; + reuse_check_delay = "inf".into() + } + + child_pid = match unsafe { fork() } { + Ok(ForkResult::Parent { child }) => child, + Ok(ForkResult::Child) => { + try_setsid(); + if unshare_succeeded { + restore_capabilities() + } + if let Err(err) = create_tmp_dirs(&tmp_dirs) { + eprintln!("Failed to create tmp dir: {err}"); exit(1) } - } else { spawn(move || waitpid(child_pid, None) ); } - - let mut exit_code = 143; - if !is_mount_only { - cfg_if! { - if #[cfg(feature = "appimage")] { - let run = tmp_dir.join("AppRun"); - if !run.is_file() { - eprintln!("AppRun not found: {:?}", run); - remove_tmp_dirs(tmp_dirs); - exit(1) - } - env::set_var("ARGV0", arg0); - env::set_var("APPDIR", &tmp_dir); - env::set_var("APPIMAGE", self_exe); - env::set_var("APPOFFSET", format!("{runtime_size}")); - } else { - let run = tmp_dir.join("static").join("bash"); - if !run.is_file() { - eprintln!("Static bash not found: {:?}", run); - remove_tmp_dirs(tmp_dirs); - exit(1) - } - exec_args.insert(0, format!("{}/Run.sh", tmp_dir.display())); - env::set_var("ARG0", arg0); - env::set_var("RUNDIR", &tmp_dir); - env::set_var("RUNIMAGE", self_exe); - env::set_var("RUNOFFSET", format!("{runtime_size}")); - } + unsafe { libc::dup2(libc::STDERR_FILENO, libc::STDOUT_FILENO) }; + if is_extract_run { + extract_image(&embed, &image, tmp_dir, is_extract_run, None) + } else { + mount_image(&embed, &image, tmp_dir, uid, gid) + } + unreachable!() + } + Err(err) => { + eprintln!("Fork error: {err}"); + exit(1) + } + }; + + if is_extract_run { + if let Err(err) = waitpid(child_pid, None) { + eprintln!("Failed to extract image: {err}"); + remove_tmp_dirs(&tmp_dirs, unshare_succeeded); + exit(1) + } + } else if !wait_mount(child_pid, &tmp_dir, Duration::from_secs(1)) { + remove_tmp_dirs(&tmp_dirs, unshare_succeeded); + let mut cmd = Command::new(self_exe); + if !unshare_succeeded && !is_unshare { + eprintln!("Trying to unshare..."); + cmd.env(format!("{ENV_NAME}_UNSHARE"), "1"); + if drop_caps_on_fallback { + cmd.env(format!("{ENV_NAME}_UNSHARE"), "2"); } - env::set_var("OWD", getcwd().unwrap()); + } else { + check_extract!(is_mount_only, uruntime_extract, self_exe, { + eprintln!("Trying to extract and run..."); + if is_unshare && !unshare_succeeded { + cmd.env_remove(format!("{ENV_NAME}_UNSHARE")) + .env_remove(format!("{ENV_NAME}_UNSHARE_ROOT")) + .env_remove(format!("{ENV_NAME}_UNSHARE_UID")) + .env_remove(format!("{ENV_NAME}_UNSHARE_GID")); + } + cmd.env(format!("{ENV_NAME}_EXTRACT_AND_RUN"), "1"); + }); + } + let err = cmd.args(&exec_args).exec(); + eprintln!("Failed to execute {:?}: {err}", self_exe); + exit(1) + } + if is_remp_mount && !is_extract_run { + if let Err(err) = write_mount_pid_file(&tmp_dir, child_pid, unshare_succeeded) { + eprintln!("Warning: failed to write PID file: {err}"); + } + } + } - try_set_portable("home", portable_home); - try_set_portable("share", portable_share); - try_set_portable("config", portable_config); + if is_mount_only { + if unshare_succeeded && (!is_tmpdir_exists || is_unshare_remp) { + println!("/proc/{child_pid}/root{}", tmp_dir.display()) + } else { + println!("{}", tmp_dir.display()) + } + } - let mut cmd = Command::new(run.canonicalize().unwrap()) - .args(&exec_args).spawn().unwrap(); - let pid = Pid::from_raw(cmd.id() as i32); + let mut exit_code = 0; + if !is_mount_only { + cfg_if! { + if #[cfg(feature = "appimage")] { + let run = tmp_dir.join("AppRun"); + if !run.is_file() { + eprintln!("AppRun not found: {:?}", run); + remove_tmp_dirs(&tmp_dirs, unshare_succeeded); + exit(1) + } + env::set_var("ARGV0", arg0); + env::set_var("APPDIR", &tmp_dir); + env::set_var("APPIMAGE", self_exe); + env::set_var("APPOFFSET", format!("{runtime_size}")); + } else { + let run = tmp_dir.join("static").join("bash"); + if !run.is_file() { + eprintln!("Static bash not found: {:?}", run); + remove_tmp_dirs(&tmp_dirs, unshare_succeeded); + exit(1) + } + exec_args.insert(0, format!("{}/Run.sh", tmp_dir.display())); + env::set_var("ARG0", arg0); + env::set_var("RUNDIR", &tmp_dir); + env::set_var("RUNIMAGE", self_exe); + env::set_var("RUNOFFSET", format!("{runtime_size}")); + } + } + env::set_var("OWD", getcwd().unwrap_or_default()); + + try_set_portable_dir(portable_share, "XDG_DATA_HOME", Some(".local/share")); + try_set_portable_dir(portable_config, "XDG_CONFIG_HOME", Some(".config")); + try_set_portable_dir(portable_cache, "XDG_CACHE_HOME", Some(".cache")); + try_set_portable_dir(portable_home, "HOME", None); + + let mut run_command = Command::new(run.canonicalize().unwrap_or(run.clone())); + if should_drop_capabilities(unshare_succeeded, drop_caps) { + let last_cap = last_capability(); + unsafe { + run_command.pre_exec(move || drop_capabilities(last_cap)); + } + } - spawn(move || signals_handler(pid, false) ); + remove_runtime_separator(&mut exec_args); + match run_command.args(&exec_args).spawn() { + Ok(mut run_child) => { + let pid = Pid::from_raw(run_child.id() as i32); + let tmp_dir_clone = tmp_dir.clone(); + spawn(move || signals_handler(pid, &tmp_dir_clone, true, false)); - if let Ok(status) = cmd.wait() { + if let Ok(status) = run_child.wait() { if let Some(code) = status.code() { exit_code = code } } - } else if !is_tmpdir_exists { - spawn(move || signals_handler(child_pid, false) ); - wait_pid_exit(child_pid, None); - } else { exit(0) } - - if is_tmpdir_exists { exit(exit_code) } else { - match unsafe { fork() } { - Ok(ForkResult::Parent { child: _ }) => { exit(exit_code) } - Ok(ForkResult::Child) => { - try_setsid(); - spawn(move || signals_handler(child_pid, true) ); - - let is_mount = !is_extract_run && !is_mount_only; - let reuse_check_delay = parse_reuse_check_delay(&reuse_check_delay); - - if is_extract_run { - if !is_noclenup && reuse_check_delay.is_some() { - wait_dir_notuse(&tmp_dir,None, reuse_check_delay, true); - let _ = remove_dir_all(&tmp_dir); - } - } else if !is_remp_mount && is_mount { - wait_dir_notuse(&tmp_dir, None, None, false); - let _ = kill(child_pid, Signal::SIGTERM); - } else if is_remp_mount && is_mount && reuse_check_delay.is_some() { - wait_dir_notuse(&tmp_dir, None, reuse_check_delay, true); - let _ = kill(child_pid, Signal::SIGTERM); - } - if is_mount { - wait_pid_exit(child_pid, Some(Duration::from_secs(1))); - } - remove_tmp_dirs(tmp_dirs); - exit(0) - } - Err(err) => { - eprintln!("Fork error: {err}"); - exit(1) - } - } + } + Err(err) => { + eprintln!("Failed to execute {:?}: {err}", run); + exit_code = 1 } } - Ok(ForkResult::Child) => { - if !is_tmpdir_exists { - try_setsid(); + } else if !is_tmpdir_exists { + let tmp_dir_clone = tmp_dir.clone(); + spawn(move || signals_handler(child_pid, &tmp_dir_clone, false, false)); + wait_pid_exit(child_pid, None); + } else { + exit(exit_code) + } - if let Err(err) = create_tmp_dirs(tmp_dirs) { - eprintln!("Failed to create tmp dir: {err}"); - exit(1) + if is_tmpdir_exists { + exit(exit_code) + } else { + match unsafe { fork() } { + Ok(ForkResult::Parent { child: _ }) => exit(exit_code), + Ok(ForkResult::Child) => { + try_setsid(); + if unshare_succeeded { + restore_capabilities() } - unsafe { libc::dup2(libc::STDERR_FILENO, libc::STDOUT_FILENO) }; + let tmp_dir_clone = tmp_dir.clone(); + spawn(move || signals_handler(child_pid, &tmp_dir_clone, false, true)); + + let is_mount = !is_extract_run && !is_mount_only; + let reuse_check_delay = parse_reuse_check_delay(&reuse_check_delay); if is_extract_run { - extract_image(&embed, &image, tmp_dir, is_extract_run, None) - } else { - mount_image(&embed, &image, tmp_dir) + if !is_noclenup && reuse_check_delay.is_some() { + wait_dir_notuse(&tmp_dir, None, reuse_check_delay, true); + let _ = remove_dir_all(&tmp_dir); + } + } else if !is_remp_mount && is_mount { + wait_dir_notuse(&tmp_dir, None, None, false); + try_unmount(Some(child_pid), &tmp_dir); + } else if is_remp_mount && is_mount && reuse_check_delay.is_some() { + wait_dir_notuse(&tmp_dir, None, reuse_check_delay, true); + try_unmount(Some(child_pid), &tmp_dir); } - } else { exit(0) } - } - Err(err) => { - eprintln!("Fork error: {err}"); - exit(1) + if is_mount { + wait_pid_exit(child_pid, Some(Duration::from_secs(1))); + } + remove_tmp_dirs(&tmp_dirs, unshare_succeeded); + exit(0) + } + Err(err) => { + eprintln!("Fork error: {err}"); + exit(1) + } } } } + +#[cfg(test)] +mod runtime_elf_tests; diff --git a/src/runtime_elf_tests.rs b/src/runtime_elf_tests.rs new file mode 100644 index 0000000..7d0cde7 --- /dev/null +++ b/src/runtime_elf_tests.rs @@ -0,0 +1,166 @@ +use super::{ + embedded_unshare_policy, environment_drop_caps_policy, fallback_should_drop_capabilities, + get_image, get_runtime, get_section_data, is_runtime_option, parse_unshare_cli_options, + remove_runtime_separator, should_drop_capabilities, UnshareCliOptions, ARG_PFX, +}; +use crate::elf_layout::tests::{fixture, Endian}; +use std::io::Write; +use tempfile::NamedTempFile; + +#[test] +fn embedded_unshare_modes_distinguish_explicit_and_fallback_capability_drop() { + assert_eq!(embedded_unshare_policy("=0"), (false, false, false)); + assert_eq!(embedded_unshare_policy("=1"), (true, false, false)); + assert_eq!(embedded_unshare_policy("=2"), (true, true, false)); + assert_eq!(embedded_unshare_policy("=3"), (false, false, true)); + assert_eq!(embedded_unshare_policy("=invalid"), (false, false, false)); + assert_eq!(environment_drop_caps_policy("1"), (true, false, false)); + assert_eq!(environment_drop_caps_policy("2"), (true, true, false)); + assert_eq!(environment_drop_caps_policy("3"), (false, false, true)); + assert_eq!( + environment_drop_caps_policy("invalid"), + (false, false, false) + ); + + assert!(!fallback_should_drop_capabilities(false, true)); + assert!(fallback_should_drop_capabilities(true, true)); + assert!(!fallback_should_drop_capabilities(true, false)); + assert!(should_drop_capabilities(true, true)); + assert!(!should_drop_capabilities(false, true)); + assert!(!should_drop_capabilities(true, false)); + + let readme = include_str!("../README.md"); + assert!(readme.contains("| `3` | Do not enable `unshare` in advance")); + assert!(readme.contains("capabilities only when `unshare` is entered automatically")); + assert!(readme.contains("_UNSHARE=3")); + assert!(readme.contains("---unshare-fallback-drop-caps")); +} + +#[test] +fn runtime_option_matching_is_exact_and_allocation_free() { + let valid = format!("--{ARG_PFX}-version"); + let unshare = format!("--{ARG_PFX}-unshare"); + let extra = format!("--{ARG_PFX}-version-extra"); + assert!(is_runtime_option(&valid, "version")); + assert!(is_runtime_option(&unshare, "unshare")); + assert!(!is_runtime_option("--other-version", "version")); + assert!(!is_runtime_option(&extra, "version")); + assert!(!is_runtime_option("runtime-version", "version")); +} + +#[test] +fn unshare_cli_options_are_composable_and_removed_before_application_launch() { + let mut args = vec![ + "--appimage-unshare-uid".into(), + "1000".into(), + "--appimage-unshare-gid=1001".into(), + "--appimage-unshare-drop-caps".into(), + "--application-option".into(), + ]; + let options = parse_unshare_cli_options(&mut args, "appimage").unwrap(); + assert!(options.enable); + assert!(options.drop_caps); + assert!(!options.drop_caps_on_fallback); + assert_eq!(options.uid.as_deref(), Some("1000")); + assert_eq!(options.gid.as_deref(), Some("1001")); + assert_eq!(args, ["--application-option"]); + + let mut args = vec![ + "--runtime-unshare-root".into(), + "--runtime-unshare-drop-caps".into(), + ]; + let options = parse_unshare_cli_options(&mut args, "runtime").unwrap(); + assert!(options.root && options.enable && options.drop_caps); + assert!(args.is_empty()); +} + +#[test] +fn runtime_separator_survives_parsing_but_is_removed_before_application_launch() { + let mut args = vec![ + "--appimage-unshare".into(), + "--".into(), + "--appimage-version".into(), + ]; + let options = parse_unshare_cli_options(&mut args, "appimage").unwrap(); + assert!(options.enable); + assert_eq!(args, ["--", "--appimage-version"]); + + // A fallback re-exec parses the same argv again; the separator must still + // protect application arguments until the final application launch. + assert_eq!( + parse_unshare_cli_options(&mut args, "appimage").unwrap(), + UnshareCliOptions::default() + ); + assert_eq!(args, ["--", "--appimage-version"]); + + remove_runtime_separator(&mut args); + assert_eq!(args, ["--appimage-version"]); +} + +#[test] +fn fallback_only_unshare_cli_mode_combines_with_explicit_unshare_by_prioritizing_drop() { + let mut args = vec!["--appimage-unshare-fallback-drop-caps".into()]; + let options = parse_unshare_cli_options(&mut args, "appimage").unwrap(); + assert!(!options.enable); + assert!(options.drop_caps_on_fallback); + + let mut args = vec![ + "--appimage-unshare".into(), + "--appimage-unshare-fallback-drop-caps".into(), + ]; + let options = parse_unshare_cli_options(&mut args, "appimage").unwrap(); + assert!(options.enable && options.drop_caps); + assert!(!options.drop_caps_on_fallback); + + let mut args = vec![ + "--appimage-unshare-root".into(), + "--appimage-unshare-uid=1000".into(), + "--appimage-unshare-gid=1001".into(), + ]; + let options = parse_unshare_cli_options(&mut args, "appimage").unwrap(); + assert!(options.root && options.enable); + assert_eq!(options.uid.as_deref(), Some("1000")); + assert_eq!(options.gid.as_deref(), Some("1001")); + + for mut args in [ + vec!["--appimage-unshare-uid".into()], + vec!["--appimage-unshare-gid=not-a-number".into()], + ] { + let original = args.clone(); + assert!(parse_unshare_cli_options(&mut args, "appimage").is_err()); + assert_eq!(args, original); + } + + let mut ordinary_args = vec!["--application-option".into(), "value".into()]; + let original = ordinary_args.clone(); + assert_eq!( + parse_unshare_cli_options(&mut ordinary_args, "appimage").unwrap(), + UnshareCliOptions::default() + ); + assert_eq!(ordinary_args, original); +} + +#[test] +fn runtime_boundary_points_to_squashfs_and_dwarfs_magic() { + for endian in [Endian::Little, Endian::Big] { + for magic in [b"hsqs", b"DWAR"] { + let mut bytes = fixture(endian); + bytes[0x380..0x384].copy_from_slice(magic); + let mut file = NamedTempFile::new().unwrap(); + file.write_all(&bytes).unwrap(); + + let runtime = get_runtime(&file.path().to_path_buf()).unwrap(); + + assert_eq!(runtime.size, 0x380); + assert_eq!(runtime.headers_bytes.len(), 0x380); + assert_eq!(runtime.envs, "VALUE=1"); + assert_eq!( + get_section_data(&runtime.headers_bytes, ".envs").unwrap(), + "VALUE=1" + ); + let image = get_image(&runtime.path, runtime.size).unwrap(); + assert_eq!(image.is_squash, magic == b"hsqs"); + assert_eq!(image.is_dwar, magic == b"DWAR"); + } + } +} diff --git a/tests/build_support.rs b/tests/build_support.rs new file mode 100644 index 0000000..7b56702 --- /dev/null +++ b/tests/build_support.rs @@ -0,0 +1,913 @@ +#[path = "../build_support.rs"] +pub mod build_support; + +use build_support::{ + all_asset_sources, asset_urls, atomic_write, cache_relative_path, cache_version, curl_args, + digest_records, download_atomic, extract_dwarfs_wrapper, parse_digest_manifest, + prepare_assets_with, sha256_file, sha256_hex, source_cache_name, source_cache_relative_path, + stage_output, target_spec, validate_elf, Asset, AssetKind, BuildFeatures, ElfEndian, ElfType, + MAX_DOWNLOAD_SIZE, MAX_HELPER_SIZE, +}; +use xxhash_rust::xxh64::xxh64; + +const TARGETS: [(&str, &str, ElfEndian); 6] = [ + ("x86_64-unknown-linux-musl", "x86_64", ElfEndian::Little), + ("aarch64-unknown-linux-musl", "aarch64", ElfEndian::Little), + ("riscv64gc-unknown-linux-musl", "riscv64", ElfEndian::Little), + ( + "loongarch64-unknown-linux-musl", + "loongarch64", + ElfEndian::Little, + ), + ("powerpc64-unknown-linux-musl", "ppc64", ElfEndian::Big), + ( + "powerpc64le-unknown-linux-musl", + "ppc64le", + ElfEndian::Little, + ), +]; + +#[test] +fn exact_target_triples_map_to_release_arch_and_endian() { + for (target, arch, endian) in TARGETS { + let spec = target_spec(target).unwrap(); + assert_eq!(spec.release_arch, arch); + assert_eq!(spec.endian, endian); + } +} + +#[test] +fn powerpc_endian_variants_do_not_share_assets() { + let be = target_spec("powerpc64-unknown-linux-musl").unwrap(); + let le = target_spec("powerpc64le-unknown-linux-musl").unwrap(); + assert_eq!(be.release_arch, "ppc64"); + assert_eq!(be.endian, ElfEndian::Big); + assert_eq!(le.release_arch, "ppc64le"); + assert_eq!(le.endian, ElfEndian::Little); +} + +#[test] +fn unknown_target_has_actionable_error() { + let err = target_spec("mips64-unknown-linux-musl").unwrap_err(); + assert!(err.contains("unsupported TARGET `mips64-unknown-linux-musl`")); + assert!(err.contains("x86_64-unknown-linux-musl")); +} + +#[test] +fn urls_cover_feature_and_lite_combinations() { + for (_, arch, _) in TARGETS { + let squash_only = asset_urls( + arch, + BuildFeatures { + squashfs: true, + dwarfs: false, + lite: false, + }, + ); + assert_eq!(squash_only.len(), 3); + assert!(squash_only.iter().any(|a| a + .url + .ends_with(&format!("/squashfuse-musl-mimalloc-{arch}")))); + assert!(squash_only + .iter() + .any(|a| a.url.ends_with(&format!("/unsquashfs-{arch}")))); + assert!(squash_only + .iter() + .any(|a| a.url.ends_with(&format!("/mksquashfs-{arch}")))); + + let squash_lite = asset_urls( + arch, + BuildFeatures { + squashfs: true, + dwarfs: false, + lite: true, + }, + ); + assert_eq!(squash_lite.len(), 2); + assert!(!squash_lite.iter().any(|a| a.name == "mksquashfs")); + + let dwarfs_full = asset_urls( + arch, + BuildFeatures { + squashfs: false, + dwarfs: true, + lite: false, + }, + ); + assert_eq!(dwarfs_full.len(), 1); + assert_eq!(dwarfs_full[0].name, "dwarfs-universal"); + assert!(dwarfs_full[0] + .url + .ends_with(&format!("/dwarfs-universal-0.15.7-Linux-{arch}"))); + + let dwarfs_lite = asset_urls( + arch, + BuildFeatures { + squashfs: false, + dwarfs: true, + lite: true, + }, + ); + assert_eq!(dwarfs_lite.len(), 1); + assert_eq!(dwarfs_lite[0].name, "dwarfs-fuse-extract"); + assert!(dwarfs_lite[0] + .url + .ends_with(&format!("/dwarfs-fuse-extract-0.15.7-Linux-{arch}"))); + + let full = asset_urls( + arch, + BuildFeatures { + squashfs: true, + dwarfs: true, + lite: false, + }, + ); + assert_eq!(full.len(), 4); + let lite = asset_urls( + arch, + BuildFeatures { + squashfs: true, + dwarfs: true, + lite: true, + }, + ); + assert_eq!(lite.len(), 3); + } +} + +#[test] +fn helper_urls_use_r2_releases() { + let urls = asset_urls( + "x86_64", + BuildFeatures { + squashfs: true, + dwarfs: false, + lite: false, + }, + ); + assert!(urls.iter().any(|a| a.url.contains("/v0.6.3.r2/"))); + assert!(urls.iter().any(|a| a.url.contains("/v4.7.5.r2/"))); +} + +#[test] +fn build_uses_only_out_dir_staging_and_target_specific_include_path() { + let build_rs = include_str!("../build.rs"); + let main_rs = include_str!("../src/main.rs"); + assert!(build_rs.contains("OUT_DIR")); + assert!(build_rs.contains("URUNTIME_HELPER_DIR")); + assert!(build_rs.contains("prepare_assets_with")); + assert!(!build_rs.contains("symlink")); + assert!(!build_rs.contains("project_path.join(\"assets\")")); + assert!(main_rs.contains("env!(\"URUNTIME_HELPER_DIR\")")); + assert!(!main_rs.contains("../assets/")); +} + +#[test] +fn cache_version_changes_when_any_helper_version_changes() { + let current = cache_version("0.6.3.r2", "4.7.5.r2", "0.15.7"); + assert_eq!( + current, + "squashfuse-0.6.3.r2_squashfs-tools-4.7.5.r2_dwarfs-0.15.7" + ); + assert_ne!(current, cache_version("0.6.3.r1", "4.7.5.r2", "0.15.7")); + assert_ne!(current, cache_version("0.6.3.r2", "4.7.5.r1", "0.15.7")); + assert_ne!(current, cache_version("0.6.3.r2", "4.7.5.r2", "0.15.6")); +} + +#[test] +fn cache_path_contains_release_arch_and_all_versions() { + let be = target_spec("powerpc64-unknown-linux-musl").unwrap(); + let le = target_spec("powerpc64le-unknown-linux-musl").unwrap(); + assert_eq!( + cache_relative_path(be), + std::path::PathBuf::from( + "assets-ppc64/squashfuse-0.6.3.r2_squashfs-tools-4.7.5.r2_dwarfs-0.15.7" + ) + ); + assert_eq!( + cache_relative_path(le), + std::path::PathBuf::from( + "assets-ppc64le/squashfuse-0.6.3.r2_squashfs-tools-4.7.5.r2_dwarfs-0.15.7" + ) + ); +} + +fn synthetic_elf(machine: u16, endian: ElfEndian) -> Vec { + let mut elf = vec![0u8; 256]; + elf[..4].copy_from_slice(b"\x7fELF"); + elf[4] = 2; + elf[5] = match endian { + ElfEndian::Little => 1, + ElfEndian::Big => 2, + }; + elf[6] = 1; + write_u16(&mut elf, 16, 2, endian); // ET_EXEC (DwarFS payload) + write_u16(&mut elf, 18, machine, endian); + write_u32(&mut elf, 20, 1, endian); + write_u64(&mut elf, 24, 0x400080, endian); + write_u64(&mut elf, 32, 64, endian); + write_u16(&mut elf, 52, 64, endian); + write_u16(&mut elf, 54, 56, endian); + write_u16(&mut elf, 56, 1, endian); + write_u32(&mut elf, 64, 1, endian); // PT_LOAD + write_u32(&mut elf, 68, 5, endian); // PF_R | PF_X + write_u64(&mut elf, 72, 128, endian); + write_u64(&mut elf, 80, 0x400080, endian); + write_u64(&mut elf, 88, 0x400080, endian); + write_u64(&mut elf, 96, 64, endian); + write_u64(&mut elf, 104, 64, endian); + write_u64(&mut elf, 112, 0x1000, endian); + elf[128..160].copy_from_slice(b"synthetic DwarFS payload fixture"); + elf +} + +fn write_u16(bytes: &mut [u8], offset: usize, value: u16, endian: ElfEndian) { + let encoded = match endian { + ElfEndian::Little => value.to_le_bytes(), + ElfEndian::Big => value.to_be_bytes(), + }; + bytes[offset..offset + 2].copy_from_slice(&encoded); +} + +fn write_u32(bytes: &mut [u8], offset: usize, value: u32, endian: ElfEndian) { + let encoded = match endian { + ElfEndian::Little => value.to_le_bytes(), + ElfEndian::Big => value.to_be_bytes(), + }; + bytes[offset..offset + 4].copy_from_slice(&encoded); +} + +fn write_u64(bytes: &mut [u8], offset: usize, value: u64, endian: ElfEndian) { + let encoded = match endian { + ElfEndian::Little => value.to_le_bytes(), + ElfEndian::Big => value.to_be_bytes(), + }; + bytes[offset..offset + 8].copy_from_slice(&encoded); +} + +fn wrapper(payload: &[u8]) -> Vec { + let compressed = zstd::stream::encode_all(payload, 3).unwrap(); + let mut wrapper = b"synthetic wrapper prefix".to_vec(); + wrapper.extend_from_slice(&compressed); + wrapper.extend_from_slice(b"SQUEEZE!"); + wrapper.extend_from_slice(&(payload.len() as u64).to_le_bytes()); + wrapper.extend_from_slice(&(compressed.len() as u64).to_le_bytes()); + wrapper.extend_from_slice(&xxh64(payload, 0).to_le_bytes()); + wrapper +} + +#[test] +fn valid_wrapper_decodes_to_exact_elf_payload() { + let spec = target_spec("x86_64-unknown-linux-musl").unwrap(); + let payload = synthetic_elf(spec.elf_machine, spec.endian); + assert_eq!( + extract_dwarfs_wrapper(&wrapper(&payload), spec, 1024).unwrap(), + payload + ); +} + +#[test] +fn truncated_wrapper_is_rejected() { + let spec = target_spec("x86_64-unknown-linux-musl").unwrap(); + let err = extract_dwarfs_wrapper(b"too short", spec, 1024).unwrap_err(); + assert!(err.contains("truncated")); +} + +#[test] +fn bad_wrapper_magic_is_rejected() { + let spec = target_spec("x86_64-unknown-linux-musl").unwrap(); + let payload = synthetic_elf(spec.elf_machine, spec.endian); + let mut input = wrapper(&payload); + let trailer = input.len() - 32; + input[trailer..trailer + 8].copy_from_slice(b"NOTMAGIC"); + let err = extract_dwarfs_wrapper(&input, spec, 1024).unwrap_err(); + assert!(err.contains("magic")); +} + +#[test] +fn impossible_compressed_size_is_rejected() { + let spec = target_spec("x86_64-unknown-linux-musl").unwrap(); + let payload = synthetic_elf(spec.elf_machine, spec.endian); + let mut input = wrapper(&payload); + let csize = input.len() - 16; + input[csize..csize + 8].copy_from_slice(&u64::MAX.to_le_bytes()); + let err = extract_dwarfs_wrapper(&input, spec, 1024).unwrap_err(); + assert!(err.contains("compressed size")); +} + +#[test] +fn checksum_mismatch_is_rejected() { + let spec = target_spec("x86_64-unknown-linux-musl").unwrap(); + let payload = synthetic_elf(spec.elf_machine, spec.endian); + let mut input = wrapper(&payload); + let checksum = input.len() - 8; + input[checksum..].copy_from_slice(&0u64.to_le_bytes()); + let err = extract_dwarfs_wrapper(&input, spec, 1024).unwrap_err(); + assert!(err.contains("XXH64")); +} + +#[test] +fn oversized_declared_output_is_rejected_before_decode() { + let spec = target_spec("x86_64-unknown-linux-musl").unwrap(); + let payload = synthetic_elf(spec.elf_machine, spec.endian); + let mut input = wrapper(&payload); + let usize_field = input.len() - 24; + input[usize_field..usize_field + 8].copy_from_slice(&2048u64.to_le_bytes()); + let err = extract_dwarfs_wrapper(&input, spec, 1024).unwrap_err(); + assert!(err.contains("exceeds limit")); +} + +#[test] +fn exact_uncompressed_size_is_required() { + let spec = target_spec("x86_64-unknown-linux-musl").unwrap(); + let payload = synthetic_elf(spec.elf_machine, spec.endian); + let mut input = wrapper(&payload); + let usize_field = input.len() - 24; + input[usize_field..usize_field + 8] + .copy_from_slice(&((payload.len() + 1) as u64).to_le_bytes()); + let err = extract_dwarfs_wrapper(&input, spec, 1024).unwrap_err(); + assert!(err.contains("uncompressed size")); +} + +#[test] +fn elf_endian_and_machine_must_match_target() { + let x86 = target_spec("x86_64-unknown-linux-musl").unwrap(); + let wrong_endian = synthetic_elf(x86.elf_machine, ElfEndian::Big); + assert!(extract_dwarfs_wrapper(&wrapper(&wrong_endian), x86, 1024) + .unwrap_err() + .contains("endian")); + + let wrong_machine = synthetic_elf(183, ElfEndian::Little); + assert!(extract_dwarfs_wrapper(&wrapper(&wrong_machine), x86, 1024) + .unwrap_err() + .contains("machine")); +} + +#[test] +fn elf_validation_rejects_short_fake_header_and_bad_program_bounds() { + let spec = target_spec("x86_64-unknown-linux-musl").unwrap(); + let mut short = vec![0u8; 20]; + short[..6].copy_from_slice(b"\x7fELF\x02\x01"); + assert!(validate_elf(&short, spec, ElfType::StaticExec) + .unwrap_err() + .contains("truncated")); + + let mut bad_bounds = synthetic_elf(spec.elf_machine, spec.endian); + write_u64(&mut bad_bounds, 32, u64::MAX, spec.endian); + assert!(validate_elf(&bad_bounds, spec, ElfType::StaticExec) + .unwrap_err() + .contains("program header")); +} + +#[test] +fn elf_validation_rejects_non_executable_load_interpreter_and_needed_library() { + let spec = target_spec("x86_64-unknown-linux-musl").unwrap(); + let mut no_exec = synthetic_elf(spec.elf_machine, spec.endian); + write_u32(&mut no_exec, 68, 4, spec.endian); + assert!(validate_elf(&no_exec, spec, ElfType::StaticExec) + .unwrap_err() + .contains("executable PT_LOAD")); + + let mut interpreter = synthetic_elf(spec.elf_machine, spec.endian); + write_u32(&mut interpreter, 64, 3, spec.endian); + assert!(validate_elf(&interpreter, spec, ElfType::StaticExec) + .unwrap_err() + .contains("PT_INTERP")); + + let mut dynamic = synthetic_elf(spec.elf_machine, spec.endian); + dynamic.resize(320, 0); + write_u16(&mut dynamic, 56, 2, spec.endian); + write_u32(&mut dynamic, 120, 2, spec.endian); // PT_DYNAMIC + write_u64(&mut dynamic, 128, 256, spec.endian); + write_u64(&mut dynamic, 152, 32, spec.endian); + write_u64(&mut dynamic, 160, 32, spec.endian); + write_u64(&mut dynamic, 256, 1, spec.endian); // DT_NEEDED + assert!(validate_elf(&dynamic, spec, ElfType::StaticExec) + .unwrap_err() + .contains("DT_NEEDED")); +} + +#[test] +fn exact_executable_type_is_enforced() { + let spec = target_spec("x86_64-unknown-linux-musl").unwrap(); + let mut elf = synthetic_elf(spec.elf_machine, spec.endian); + assert!(validate_elf(&elf, spec, ElfType::StaticPie).is_err()); + write_u16(&mut elf, 16, 3, spec.endian); + validate_elf(&elf, spec, ElfType::StaticPie).unwrap(); + assert!(validate_elf(&elf, spec, ElfType::StaticExec).is_err()); +} + +#[test] +fn upstream_pack_py_zstd_vector_decodes_exactly() { + let spec = target_spec("x86_64-unknown-linux-musl").unwrap(); + let wrapper = include_bytes!("fixtures/pack-py-zstd.wrapper"); + let payload = include_bytes!("fixtures/pack-py-payload.elf"); + assert_eq!( + extract_dwarfs_wrapper(wrapper, spec, MAX_HELPER_SIZE).unwrap(), + payload + ); +} + +#[test] +fn helper_source_inventory_is_derived_from_pinned_versions_for_all_30_assets() { + let sources = all_asset_sources(); + assert_eq!(sources.len(), 30); + let keys: std::collections::BTreeSet<_> = sources + .iter() + .map(|source| (source.target.release_arch, source.name)) + .collect(); + assert_eq!(keys.len(), 30); + assert!(sources.iter().all(|source| { + source.url.contains("0.15.7") + || source.url.contains("0.6.3.r2") + || source.url.contains("4.7.5.r2") + })); +} + +#[test] +fn manifest_pins_every_remote_asset_for_all_arches() { + let records = digest_records().unwrap(); + assert_eq!(records.len(), 30); + assert!(records.windows(2).all(|pair| { + (pair[0].arch.as_str(), pair[0].name.as_str()) + < (pair[1].arch.as_str(), pair[1].name.as_str()) + })); + let malformed = "x86_64\thelper\tnot-a-hash\talso-bad\n"; + assert!(parse_digest_manifest(malformed) + .unwrap_err() + .contains("SHA-256")); + let duplicate = format!( + "x86_64\thelper\t{}\t{}\nx86_64\thelper\t{}\t{}\n", + "0".repeat(64), + "1".repeat(64), + "2".repeat(64), + "3".repeat(64) + ); + assert!(parse_digest_manifest(&duplicate) + .unwrap_err() + .contains("duplicate")); + + let mut sources = std::collections::BTreeSet::new(); + for (_, arch, _) in TARGETS { + let assets = asset_urls( + arch, + BuildFeatures { + squashfs: true, + dwarfs: true, + lite: false, + }, + ); + assert_eq!(assets.len(), 4); + for asset in assets { + assert_eq!(asset.source_sha256.len(), 64); + assert!(asset.source_sha256.bytes().all(|b| b.is_ascii_hexdigit())); + assert_eq!(asset.payload_sha256.len(), 64); + sources.insert(asset.url); + } + let lite = asset_urls( + arch, + BuildFeatures { + squashfs: false, + dwarfs: true, + lite: true, + }, + ); + assert_eq!(lite.len(), 1); + assert_eq!(lite[0].source_sha256.len(), 64); + assert_eq!(lite[0].payload_sha256.len(), 64); + sources.insert(lite[0].url.clone()); + } + assert_eq!(sources.len(), 30); +} + +#[test] +fn resource_limits_cover_published_assets_without_being_unbounded() { + assert_eq!(MAX_DOWNLOAD_SIZE, 8 * 1024 * 1024); + assert_eq!(MAX_HELPER_SIZE, 16 * 1024 * 1024); +} + +#[test] +fn curl_download_arguments_are_secure_and_retrying() { + let args = curl_args("https://example.invalid/helper", MAX_DOWNLOAD_SIZE); + assert_eq!( + args, + [ + "--fail", + "--location", + "--retry", + "3", + "--max-filesize", + "8388608", + "https://example.invalid/helper", + ] + ); + assert!(!args.iter().any(|arg| arg == "--insecure" || arg == "-k")); + assert!(!args.iter().any(|arg| arg == "--output" || arg == "-o")); +} + +#[test] +fn failed_wrapper_extraction_does_not_touch_existing_destination() { + let spec = target_spec("x86_64-unknown-linux-musl").unwrap(); + let dir = std::env::temp_dir().join(format!( + "uruntime-wrapper-test-{}-{}", + std::process::id(), + std::thread::current().name().unwrap_or("unnamed") + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let wrapper_path = dir.join("wrapper"); + let destination = dir.join("payload"); + std::fs::write(&wrapper_path, b"invalid").unwrap(); + std::fs::write(&destination, b"existing cache").unwrap(); + + let wrapper = std::fs::read(&wrapper_path).unwrap(); + assert!(extract_dwarfs_wrapper(&wrapper, spec, 1024).is_err()); + assert_eq!(std::fs::read(&destination).unwrap(), b"existing cache"); + let names: Vec<_> = std::fs::read_dir(&dir) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert!(!names + .iter() + .any(|name| name.to_string_lossy().contains(".part-"))); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn oversized_cache_file_is_rejected_before_reading() { + let root = test_dir("oversized-cache"); + let path = root.join("oversized"); + let file = std::fs::File::create(&path).unwrap(); + file.set_len((MAX_DOWNLOAD_SIZE + 1) as u64).unwrap(); + let err = sha256_file(&path, MAX_DOWNLOAD_SIZE).unwrap_err(); + assert!(err.contains("exceeds limit")); + std::fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn atomic_write_replaces_destination_symlink_without_following_it() { + use std::os::unix::fs::symlink; + + let root = test_dir("atomic-symlink"); + let victim = root.join("victim"); + let destination = root.join("destination"); + std::fs::write(&victim, b"victim-data").unwrap(); + symlink(&victim, &destination).unwrap(); + + atomic_write(&destination, b"new-data").unwrap(); + + assert_eq!(std::fs::read(&victim).unwrap(), b"victim-data"); + assert_eq!(std::fs::read(&destination).unwrap(), b"new-data"); + assert!(!std::fs::symlink_metadata(&destination) + .unwrap() + .file_type() + .is_symlink()); + std::fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn corrupted_source_cache_is_deleted_and_reacquired() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let spec = target_spec("x86_64-unknown-linux-musl").unwrap(); + let payload = synthetic_elf(spec.elf_machine, spec.endian); + let digest = sha256_hex(&payload); + let asset = Asset { + name: "squashfuse", + url: "https://example.invalid/helper".to_string(), + kind: AssetKind::Direct, + source_sha256: digest.clone(), + payload_sha256: digest, + elf_type: ElfType::StaticExec, + }; + let root = test_dir("corrupt-cache"); + let cache = root.join("cache"); + let out = root.join("out"); + let calls = AtomicUsize::new(0); + let downloader = |_: &Asset, path: &std::path::Path| { + calls.fetch_add(1, Ordering::SeqCst); + std::fs::write(path, &payload).map_err(|err| err.to_string()) + }; + + prepare_assets_with( + &cache, + &out, + spec, + std::slice::from_ref(&asset), + &downloader, + ) + .unwrap(); + std::fs::write(cache.join(".source-squashfuse"), b"corrupt").unwrap(); + prepare_assets_with(&cache, &out, spec, &[asset], &downloader).unwrap(); + + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!( + sha256_file(&cache.join(".source-squashfuse"), MAX_DOWNLOAD_SIZE).unwrap(), + asset_source_hash(&payload) + ); + std::fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn shared_source_cache_is_reused_by_new_feature_generations() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let spec = target_spec("x86_64-unknown-linux-musl").unwrap(); + let payload = synthetic_elf(spec.elf_machine, spec.endian); + let digest = sha256_hex(&payload); + let asset = Asset { + name: "squashfuse", + url: "https://example.invalid/helper".to_string(), + kind: AssetKind::Direct, + source_sha256: digest.clone(), + payload_sha256: digest, + elf_type: ElfType::StaticExec, + }; + let root = test_dir("shared-source-cache"); + let cache = root.join("version").join("feature-generation"); + let out = root.join("out"); + std::fs::create_dir_all(cache.parent().unwrap()).unwrap(); + let shared_source = cache + .parent() + .unwrap() + .join("squashfuse-0.6.3.r2") + .join(source_cache_name(asset.name, asset.kind)); + std::fs::create_dir_all(shared_source.parent().unwrap()).unwrap(); + std::fs::write(shared_source, &payload).unwrap(); + let calls = AtomicUsize::new(0); + let downloader = |_: &Asset, _: &std::path::Path| { + calls.fetch_add(1, Ordering::SeqCst); + Err("shared source should avoid a download".to_string()) + }; + + prepare_assets_with(&cache, &out, spec, &[asset], &downloader).unwrap(); + + assert_eq!(calls.load(Ordering::SeqCst), 0); + assert!(out.join("squashfuse-zst").is_file()); + std::fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn helper_source_caches_are_versioned_per_upstream_project() { + let target = target_spec("x86_64-unknown-linux-musl").unwrap(); + assert_eq!( + source_cache_relative_path(target, "squashfuse", AssetKind::Direct).unwrap(), + std::path::PathBuf::from("assets-x86_64/squashfuse-0.6.3.r2/squashfuse") + ); + assert_eq!( + source_cache_relative_path(target, "mksquashfs", AssetKind::Direct).unwrap(), + std::path::PathBuf::from("assets-x86_64/squashfs-tools-4.7.5.r2/mksquashfs") + ); + assert_eq!( + source_cache_relative_path(target, "dwarfs-universal", AssetKind::DwarfsWrapper).unwrap(), + std::path::PathBuf::from("assets-x86_64/dwarfs-0.15.7/dwarfs-universal-wrapper") + ); +} + +#[test] +fn wrapper_sources_have_distinct_shared_cache_names() { + assert_eq!( + source_cache_name("dwarfs-universal", AssetKind::DwarfsWrapper), + "dwarfs-universal-wrapper" + ); + assert_eq!( + source_cache_name("squashfuse", AssetKind::Direct), + "squashfuse" + ); +} + +#[test] +fn partial_generation_recovers_and_level_22_zst_is_byte_equal_to_raw_elf() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let spec = target_spec("x86_64-unknown-linux-musl").unwrap(); + let payload = synthetic_elf(spec.elf_machine, spec.endian); + let digest = sha256_hex(&payload); + let asset = Asset { + name: "squashfuse", + url: "https://example.invalid/helper".to_string(), + kind: AssetKind::Direct, + source_sha256: digest.clone(), + payload_sha256: digest, + elf_type: ElfType::StaticExec, + }; + let root = test_dir("partial-generation"); + let cache = root.join("cache"); + let out = root.join("out"); + let calls = AtomicUsize::new(0); + let downloader = |_: &Asset, path: &std::path::Path| { + calls.fetch_add(1, Ordering::SeqCst); + std::fs::write(path, &payload).map_err(|err| err.to_string()) + }; + + prepare_assets_with( + &cache, + &out, + spec, + std::slice::from_ref(&asset), + &downloader, + ) + .unwrap(); + std::fs::remove_file(cache.join("squashfuse-zst")).unwrap(); + prepare_assets_with(&cache, &out, spec, &[asset], &downloader).unwrap(); + + assert_eq!(calls.load(Ordering::SeqCst), 1); + let raw = std::fs::read(cache.join("squashfuse")).unwrap(); + let decoded = + zstd::stream::decode_all(std::fs::File::open(cache.join("squashfuse-zst")).unwrap()) + .unwrap(); + assert_eq!(decoded, raw); + std::fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn concurrent_generation_uses_one_locked_transaction() { + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Barrier, + }; + + let spec = target_spec("x86_64-unknown-linux-musl").unwrap(); + let payload = Arc::new(synthetic_elf(spec.elf_machine, spec.endian)); + let digest = sha256_hex(&payload); + let asset = Asset { + name: "squashfuse", + url: "https://example.invalid/helper".to_string(), + kind: AssetKind::Direct, + source_sha256: digest.clone(), + payload_sha256: digest, + elf_type: ElfType::StaticExec, + }; + let root = test_dir("concurrent-generation"); + let cache = root.join("cache"); + let calls = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(Barrier::new(2)); + let mut threads = Vec::new(); + for n in 0..2 { + let payload = Arc::clone(&payload); + let calls = Arc::clone(&calls); + let barrier = Arc::clone(&barrier); + let cache = cache.clone(); + let out = root.join(format!("out-{n}")); + let asset = asset.clone(); + threads.push(std::thread::spawn(move || { + barrier.wait(); + prepare_assets_with(&cache, &out, spec, &[asset], &|_, path| { + calls.fetch_add(1, Ordering::SeqCst); + std::thread::sleep(std::time::Duration::from_millis(75)); + std::fs::write(path, payload.as_slice()).map_err(|err| err.to_string()) + }) + })); + } + for thread in threads { + thread.join().unwrap().unwrap(); + } + assert_eq!(calls.load(Ordering::SeqCst), 1); + std::fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn sequential_a_b_a_and_concurrent_outputs_are_isolated() { + let root = test_dir("output-isolation"); + let a = root.join("cache-a"); + let b = root.join("cache-b"); + std::fs::create_dir_all(&a).unwrap(); + std::fs::create_dir_all(&b).unwrap(); + std::fs::write(a.join("helper-zst"), b"architecture-a").unwrap(); + std::fs::write(b.join("helper-zst"), b"architecture-b").unwrap(); + let shared_out = root.join("sequential-out"); + stage_output(&a, &shared_out, &["helper-zst"]).unwrap(); + assert_eq!( + std::fs::read(shared_out.join("helper-zst")).unwrap(), + b"architecture-a" + ); + stage_output(&b, &shared_out, &["helper-zst"]).unwrap(); + assert_eq!( + std::fs::read(shared_out.join("helper-zst")).unwrap(), + b"architecture-b" + ); + stage_output(&a, &shared_out, &["helper-zst"]).unwrap(); + assert_eq!( + std::fs::read(shared_out.join("helper-zst")).unwrap(), + b"architecture-a" + ); + + let out_a = root.join("out-a"); + let out_b = root.join("out-b"); + let a2 = a.clone(); + let b2 = b.clone(); + let ta = std::thread::spawn(move || stage_output(&a2, &out_a, &["helper-zst"])); + let tb = std::thread::spawn(move || stage_output(&b2, &out_b, &["helper-zst"])); + ta.join().unwrap().unwrap(); + tb.join().unwrap().unwrap(); + assert_eq!( + std::fs::read(root.join("out-a/helper-zst")).unwrap(), + b"architecture-a" + ); + assert_eq!( + std::fs::read(root.join("out-b/helper-zst")).unwrap(), + b"architecture-b" + ); + std::fs::remove_dir_all(root).unwrap(); +} + +fn asset_source_hash(payload: &[u8]) -> String { + sha256_hex(payload) +} + +fn test_dir(label: &str) -> std::path::PathBuf { + let path = std::env::temp_dir().join(format!( + "uruntime-{label}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).unwrap(); + path +} + +#[test] +#[ignore = "downloads and validates all upstream direct SquashFS helpers"] +fn all_upstream_direct_helpers_match_manifest_and_elf_policy() { + let cache = std::env::temp_dir().join("uruntime-direct-helper-tests-r2"); + std::fs::create_dir_all(&cache).unwrap(); + for (target, arch, _) in TARGETS { + let spec = target_spec(target).unwrap(); + for asset in asset_urls( + arch, + BuildFeatures { + squashfs: true, + dwarfs: false, + lite: false, + }, + ) { + let path = cache.join(format!("{arch}-{}", asset.name)); + if sha256_file(&path, MAX_DOWNLOAD_SIZE).ok().as_deref() != Some(&asset.source_sha256) { + let _ = std::fs::remove_file(&path); + download_atomic("curl", &asset.url, &path).unwrap(); + } + assert_eq!( + sha256_file(&path, MAX_DOWNLOAD_SIZE).unwrap(), + asset.source_sha256 + ); + let payload = std::fs::read(&path).unwrap(); + assert_eq!(sha256_hex(&payload), asset.payload_sha256); + validate_elf(&payload, spec, asset.elf_type).unwrap(); + } + } +} + +#[test] +#[ignore = "downloads and validates all upstream DwarFS wrappers"] +fn all_upstream_wrappers_decode_and_match_target_elf() { + use std::os::unix::fs::PermissionsExt; + + let cache = std::env::temp_dir().join("uruntime-dwarfs-wrapper-tests-0.15.7"); + std::fs::create_dir_all(&cache).unwrap(); + for (target, arch, _) in TARGETS { + let spec = target_spec(target).unwrap(); + for lite in [false, true] { + let asset = asset_urls( + arch, + BuildFeatures { + squashfs: false, + dwarfs: true, + lite, + }, + ) + .pop() + .unwrap(); + let wrapper_path = cache.join(format!("{arch}-{}-wrapper", asset.name)); + if sha256_file(&wrapper_path, MAX_DOWNLOAD_SIZE) + .ok() + .as_deref() + != Some(&asset.source_sha256) + { + let _ = std::fs::remove_file(&wrapper_path); + download_atomic("curl", &asset.url, &wrapper_path).unwrap(); + } + assert_eq!( + sha256_file(&wrapper_path, MAX_DOWNLOAD_SIZE).unwrap(), + asset.source_sha256 + ); + let wrapper_data = std::fs::read(&wrapper_path).unwrap(); + let decoded = extract_dwarfs_wrapper(&wrapper_data, spec, MAX_HELPER_SIZE).unwrap(); + + if arch == "x86_64" { + let mut permissions = std::fs::metadata(&wrapper_path).unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&wrapper_path, permissions).unwrap(); + let reference = cache.join(format!("{arch}-{}-reference", asset.name)); + let _ = std::fs::remove_file(&reference); + let status = std::process::Command::new(&wrapper_path) + .arg("--extract-wrapped-binary") + .arg(&reference) + .status() + .unwrap(); + assert!(status.success()); + assert_eq!(decoded, std::fs::read(&reference).unwrap()); + std::fs::remove_file(reference).unwrap(); + } + } + } +} diff --git a/tests/fixtures/pack-py-payload.elf b/tests/fixtures/pack-py-payload.elf new file mode 100644 index 0000000..3437d36 Binary files /dev/null and b/tests/fixtures/pack-py-payload.elf differ diff --git a/tests/fixtures/pack-py-zstd.wrapper b/tests/fixtures/pack-py-zstd.wrapper new file mode 100644 index 0000000..8636c63 Binary files /dev/null and b/tests/fixtures/pack-py-zstd.wrapper differ diff --git a/tests/test_ci_artifacts.py b/tests/test_ci_artifacts.py new file mode 100644 index 0000000..dc570f2 --- /dev/null +++ b/tests/test_ci_artifacts.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +import importlib.util +import json +import struct +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = ROOT / "scripts" / "ci_artifacts.py" + + +def load_module(): + spec = importlib.util.spec_from_file_location("ci_artifacts", MODULE_PATH) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {MODULE_PATH}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def make_elf(path: Path, *, machine: int, endian: str, magic: bytes, interp: bool = False, + needed: bool = False, + omit_section: str | None = None, nobits_outside: bool = False): + order = "<" if endian == "little" else ">" + names = [".envs", ".upd_info", ".sig_key", ".sha256_sig", ".digest_md5"] + if omit_section: + names.remove(omit_section) + if nobits_outside: + names.append(".bss") + shstr = b"\0" + name_offsets = {} + for name in names + [".shstrtab"]: + name_offsets[name] = len(shstr) + shstr += name.encode() + b"\0" + + phnum = 2 if interp or needed else 1 + shnum = 1 + len(names) + 1 + phoff = 64 + shstr_offset = 256 + data_offset = 384 + shoff = 512 + size = shoff + shnum * 64 + image = bytearray(size) + image[:4] = b"\x7fELF" + image[4:8] = bytes((2, 1 if endian == "little" else 2, 1, 0)) + image[8:11] = magic + struct.pack_into(order + "HHIQQQIHHHHHH", image, 16, + 3, machine, 1, 0, phoff, shoff, 0, 64, 56, phnum, 64, shnum, + shnum - 1) + struct.pack_into(order + "IIQQQQQQ", image, phoff, + 1, 5, 0, 0, 0, size, size, 0x1000) + if interp: + struct.pack_into(order + "IIQQQQQQ", image, phoff + 56, + 3, 4, data_offset, 0, 0, 8, 8, 1) + if needed: + struct.pack_into(order + "IIQQQQQQ", image, phoff + 56, + 2, 4, data_offset + 64, 0, 0, 32, 32, 8) + image[shstr_offset:shstr_offset + len(shstr)] = shstr + for index, name in enumerate(names, 1): + image[data_offset + index] = index + section_type = 8 if name == ".bss" else 1 + section_offset = size + 4096 if name == ".bss" else data_offset + index + section_size = 8192 if name == ".bss" else 1 + struct.pack_into(order + "IIQQQQIIQQ", image, shoff + index * 64, + name_offsets[name], section_type, 0, 0, section_offset, section_size, + 0, 0, 1, 0) + struct.pack_into(order + "IIQQQQIIQQ", image, shoff + (shnum - 1) * 64, + name_offsets[".shstrtab"], 3, 0, 0, shstr_offset, len(shstr), 0, 0, 1, 0) + if needed: + struct.pack_into(order + "QQQQ", image, data_offset + 64, 1, 1, 0, 0) + path.write_bytes(image) + path.chmod(0o755) + + +class ArtifactValidationTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ci = load_module() + + def populate_arch(self, root: Path, arch: str): + spec = self.ci.ARCHES[arch] + for name in self.ci.expected_artifact_names(arch): + magic = b"AI\x02" if "appimage" in name else b"RI\x02" + make_elf(root / name, machine=spec.machine, endian=spec.endian, magic=magic) + + def test_expected_manifest_has_nine_unique_files_per_arch(self): + all_names = [] + for arch in self.ci.ARCHES: + names = self.ci.expected_artifact_names(arch) + self.assertEqual(9, len(names)) + self.assertEqual(9, len(set(names))) + all_names.extend(names) + self.assertEqual(54, len(set(all_names))) + + def test_validate_arch_checks_exact_manifest_and_big_endian_elf_contract(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + self.populate_arch(root, "ppc64") + self.ci.validate_arch(root, "ppc64") + (root / "stale-file").write_text("stale") + with self.assertRaisesRegex(ValueError, "unexpected"): + self.ci.validate_arch(root, "ppc64") + + def test_validate_arch_rejects_missing_static_sections_and_wrong_magic(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + self.populate_arch(root, "aarch64") + victim = root / "uruntime-appimage-aarch64" + make_elf(victim, machine=183, endian="little", magic=b"RI\x02", + omit_section=".upd_info") + with self.assertRaisesRegex(ValueError, "magic|section"): + self.ci.validate_arch(root, "aarch64") + + def test_validate_arch_rejects_dynamic_interpreter(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + self.populate_arch(root, "x86_64") + victim = root / "uruntime-runimage-x86_64" + make_elf(victim, machine=62, endian="little", magic=b"RI\x02", interp=True) + with self.assertRaisesRegex(ValueError, "PT_INTERP"): + self.ci.validate_arch(root, "x86_64") + + def test_validate_arch_rejects_needed_shared_library(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + self.populate_arch(root, "x86_64") + victim = root / "uruntime-runimage-x86_64" + make_elf(victim, machine=62, endian="little", magic=b"RI\x02", needed=True) + with self.assertRaisesRegex(ValueError, "DT_NEEDED"): + self.ci.validate_arch(root, "x86_64") + + def test_validate_elf_rejects_oversized_files_and_extended_program_numbering(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + oversized = root / "oversized" + with oversized.open("wb") as output: + output.truncate(self.ci.MAX_ARTIFACT_SIZE + 1) + with self.assertRaisesRegex(ValueError, "size limit"): + self.ci.validate_elf(oversized, "x86_64", b"RI\x02") + + binary = root / "extended" + make_elf(binary, machine=62, endian="little", magic=b"RI\x02") + data = bytearray(binary.read_bytes()) + struct.pack_into("&2"): + noisy = ( + "#!/bin/sh\n" + "while :; do printf '0123456789abcdef0123456789abcdef'" + f"{redirect}; done\n" + ).encode() + with self.subTest(redirect=redirect), self.assertRaisesRegex( + ValueError, "output limit|signal|failed" + ): + self.ci._smoke_validated_bytes( + "x86_64", "uruntime-runimage-x86_64", noisy + ) + + def test_exclusive_staging_refuses_to_clobber(self): + with tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) / "artifact" + destination.write_bytes(b"existing") + with self.assertRaisesRegex(ValueError, "already exists|exclusive"): + self.ci._write_exclusive(destination, b"replacement") + self.assertEqual(b"existing", destination.read_bytes()) + + def test_release_aggregation_stages_the_bytes_that_were_validated_without_reopen(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "downloads" + output = Path(temporary) / "release" + root.mkdir() + for arch in self.ci.ARCHES: + artifact = root / f"uruntime-{arch}" + artifact.mkdir() + self.populate_arch(artifact, arch) + + victim = root / "uruntime-x86_64" / "uruntime-runimage-x86_64" + validated_bytes = victim.read_bytes() + original_read = self.ci._read_regular + victim_reads = 0 + + def swap_after_read(path, limit=self.ci.MAX_ARTIFACT_SIZE): + nonlocal victim_reads + data = original_read(path, limit) + if path == victim: + victim_reads += 1 + victim.write_bytes(b"unvalidated replacement") + return data + + with mock.patch.object(self.ci, "_read_regular", side_effect=swap_after_read): + self.ci.aggregate_release(root, output) + + self.assertEqual(1, victim_reads) + self.assertEqual(validated_bytes, (output / victim.name).read_bytes()) + + def test_validate_elf_rejects_bounded_metadata_dimensions(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for field_offset, value, message in ( + (60, self.ci.MAX_SECTION_HEADERS + 1, "section header"), + (56, self.ci.MAX_PROGRAM_HEADERS + 1, "program header"), + ): + binary = root / f"bounded-{field_offset}" + make_elf(binary, machine=62, endian="little", magic=b"RI\x02") + data = bytearray(binary.read_bytes()) + struct.pack_into(" 1", acquire) + self.assertIn("--method POST", acquire) + self.assertIn("--method PATCH", acquire) + self.assertIn("releases/$release_id", acquire) + self.assertIn('if [[ -n "$release_id" ]]', acquire) + self.assertIn("release_action=refreshed", acquire) + self.assertIn("release_action=created", acquire) + for field in ("target_commitish", "tag_name", "name", "body"): + self.assertIn(field, acquire) + for field in ("draft=true", "prerelease=false", "generate_release_notes=false", "make_latest=false"): + self.assertIn(field, acquire) + self.assertIn("RELEASE_MARKER", acquire) + self.assertIn("cat RELEASE_NOTES.md", acquire) + self.assertIn(".id == $id", acquire) + self.assertIn(".tag_name == $tag", acquire) + deletion = runs[delete_index] + self.assertIn("releases/$RELEASE_ID/assets?per_page=100", deletion) + self.assertIn("releases/assets/$asset_id", deletion) + self.assertIn("--paginate", deletion) + self.assertIn("--slurp", deletion) + self.assertIn(".id == $id", deletion) + self.assertIn(".tag_name == $tag", deletion) + self.assertIn(".draft == true", deletion) + self.assertIn("RELEASE_ID", runs[upload_index]) + self.assertIn("releases/$RELEASE_ID/assets", runs[upload_index]) + self.assertIn('--data-binary "@$file"', runs[upload_index]) + self.assertIn('Authorization: Bearer ', runs[upload_index]) + self.assertIn("--paginate", runs[verify_index]) + self.assertIn("--slurp", runs[verify_index]) + self.assertIn("releases/$RELEASE_ID/assets?per_page=100", runs[verify_index]) + self.assertIn("releases/$RELEASE_ID", runs[publish_index]) + self.assertIn("draft=false", runs[publish_index]) + self.assertIn("generate_release_notes=false", runs[publish_index]) + self.assertIn("make_latest=legacy", runs[publish_index]) + self.assertIn("RELEASE_MARKER", runs[publish_index]) + self.assertIn("cat RELEASE_NOTES.md", runs[publish_index]) + self.assertIn("published-release-readback.json", runs[publish_index]) + self.assertRegex(combined, r"\[\[ \$RELEASE_ID =~ \^\[0-9\]\+\$ \]\]") + + expected_id = "${{ steps.acquire_release.outputs.release_id }}" + for index in (delete_index, upload_index, verify_index, publish_index): + self.assertEqual(expected_id, steps[index]["env"]["RELEASE_ID"]) + + def test_every_cargo_command_uses_the_committed_lockfile(self): + commands = re.findall(r"\bcargo --locked [^\n]+", self.raw) + self.assertGreaterEqual(len(commands), 2) + self.assertNotRegex(self.raw, r"\bcargo\s+(?!--locked\b)") + + def test_release_shell_receives_expressions_only_through_environment(self): + release = self.jobs["release"] + for step in release["steps"]: + run = step.get("run", "") + self.assertNotIn("${{", run, step.get("name", "unnamed step")) + + def test_every_action_is_an_immutable_current_version_pin_with_comment(self): + uses = re.findall(r"^\s*uses:\s*([^\s#]+)(?:\s+#\s*(\S+))?", self.raw, re.MULTILINE) + self.assertGreaterEqual(len(uses), 5) + for action_ref, comment in uses: + action, separator, sha = action_ref.partition("@") + self.assertTrue(separator, action_ref) + self.assertRegex(sha, r"^[0-9a-f]{40}$") + self.assertIn(action, PINNED_ACTIONS) + expected_sha, expected_version = PINNED_ACTIONS[action] + self.assertEqual(expected_sha, sha) + self.assertEqual(expected_version, comment) + + +if __name__ == "__main__": + unittest.main() diff --git a/xtask/Cargo.lock b/xtask/Cargo.lock new file mode 100644 index 0000000..2e2f65a --- /dev/null +++ b/xtask/Cargo.lock @@ -0,0 +1,391 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cc" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "syn" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "xtask" +version = "0.1.0" +dependencies = [ + "fs2", + "serde_json", + "sha2", + "tempfile", + "xxhash-rust", + "zstd", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.0+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index a8f76b4..8dfacbe 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -2,4 +2,12 @@ name = "xtask" version = "0.1.0" authors = ["Aleksey Kladov "] -edition = "2018" +edition = "2021" + +[dependencies] +fs2 = "0.4.3" +sha2 = "0.10.9" +serde_json = "1.0.145" +tempfile = "3.23.0" +zstd = { version = "0.13.3", default-features = false } +xxhash-rust = { version = "0.8.18", features = ["xxh64"] } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 2bd68ae..9c139fd 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,262 +1,1258 @@ use std::{ env, - path::{Path, PathBuf}, + ffi::{OsStr, OsString}, + fs::{self, create_dir_all, File, OpenOptions}, io::{Seek, SeekFrom, Write}, + path::{Path, PathBuf}, process::{exit, Command, Stdio}, - fs::{create_dir_all, rename, OpenOptions}, }; +use fs2::FileExt; + +#[path = "../../build_support.rs"] +pub mod build_support; +use build_support::{ + source_cache_relative_path, with_cache_lock, CHECKSUM_MANIFEST, MAX_DOWNLOAD_SIZE, + ZIG_DOWNLOAD_BASE, ZIG_DOWNLOAD_MAX, ZIG_INDEX_URL, ZIG_PLATFORMS, ZIG_VERSION, +}; const BIN_NAME: &str = "uruntime"; -const TARGET_X86_64: &str = "x86_64-unknown-linux-musl"; -const TARGET_AARCH64: &str = "aarch64-unknown-linux-musl"; + +fn zig_download_url(platform: &str) -> String { + format!("{ZIG_DOWNLOAD_BASE}/{ZIG_VERSION}/zig-{platform}-{ZIG_VERSION}.tar.xz") +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct ZigPackage { + version: String, + platform: String, + url: String, + sha256: String, +} + +fn valid_sha256(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) +} + +fn parse_zig_packages(manifest: &str) -> Result, DynError> { + let mut section = ""; + let mut packages = Vec::new(); + for (index, line) in manifest.lines().enumerate() { + let line_number = index + 1; + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + if line.starts_with('[') && line.ends_with(']') { + section = line; + continue; + } + if section != "[zig]" { + continue; + } + let fields: Vec<&str> = line.split('\t').collect(); + if fields.len() != 4 || !valid_sha256(fields[3]) { + return Err(format!("invalid Zig checksum record on line {line_number}").into()); + } + packages.push(ZigPackage { + version: fields[0].to_string(), + platform: fields[1].to_string(), + url: fields[2].to_string(), + sha256: fields[3].to_string(), + }); + } + packages.sort_by(|left, right| left.platform.cmp(&right.platform)); + if packages.len() != ZIG_PLATFORMS.len() + || packages + .iter() + .map(|package| package.platform.as_str()) + .ne(ZIG_PLATFORMS) + { + return Err( + "Zig checksum inventory must contain exactly the five supported Linux hosts".into(), + ); + } + let version = packages + .first() + .ok_or("Zig checksum inventory is empty")? + .version + .clone(); + if packages.iter().any(|package| package.version != version) { + return Err("Zig checksum inventory mixes multiple versions".into()); + } + if version != ZIG_VERSION { + return Err(format!( + "Zig checksum manifest contains version {version}, but build_support.rs requires {ZIG_VERSION}; run `cargo xtask update-checksums`" + ) + .into()); + } + Ok(packages) +} + +fn zig_platform(os: &str, arch: &str) -> Option<&'static str> { + if os != "linux" { + return None; + } + match arch { + "x86_64" => Some("x86_64-linux"), + "aarch64" => Some("aarch64-linux"), + "riscv64" => Some("riscv64-linux"), + "powerpc64" if cfg!(target_endian = "little") => Some("powerpc64le-linux"), + "loongarch64" => Some("loongarch64-linux"), + _ => None, + } +} + +fn zig_packages() -> Result, DynError> { + parse_zig_packages(CHECKSUM_MANIFEST) +} + +fn zig_package(os: &str, arch: &str) -> Result { + let platform = zig_platform(os, arch).ok_or_else(|| { + format!( + "automatic Zig installation is unsupported on {arch}-{os}; set URUNTIME_ZIG to a compatible Zig binary" + ) + })?; + zig_packages()? + .into_iter() + .find(|package| package.platform == platform) + .ok_or_else(|| format!("missing Zig checksum for host platform {platform}").into()) +} type DynError = Box; -fn main() { - if let Err(e) = try_main() { - eprintln!("{}", e); - exit(-1); +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Endian { + Little, + Big, +} + +#[derive(Clone, Copy, Debug)] +struct Arch { + artifact_name: &'static str, + rust_target: &'static str, + zig_target: &'static str, + endian: Endian, +} + +const ARCHES: [Arch; 6] = [ + Arch { + artifact_name: "x86_64", + rust_target: "x86_64-unknown-linux-musl", + zig_target: "x86_64-linux-musl", + endian: Endian::Little, + }, + Arch { + artifact_name: "aarch64", + rust_target: "aarch64-unknown-linux-musl", + zig_target: "aarch64-linux-musl", + endian: Endian::Little, + }, + Arch { + artifact_name: "riscv64", + rust_target: "riscv64gc-unknown-linux-musl", + zig_target: "riscv64-linux-musl", + endian: Endian::Little, + }, + Arch { + artifact_name: "loongarch64", + rust_target: "loongarch64-unknown-linux-musl", + zig_target: "loongarch64-linux-musl", + endian: Endian::Little, + }, + Arch { + artifact_name: "ppc64", + rust_target: "powerpc64-unknown-linux-musl", + zig_target: "powerpc64-linux-musl", + endian: Endian::Big, + }, + Arch { + artifact_name: "ppc64le", + rust_target: "powerpc64le-unknown-linux-musl", + zig_target: "powerpc64le-linux-musl", + endian: Endian::Little, + }, +]; + +#[derive(Clone, Copy, Debug)] +struct Variant { + name: &'static str, + description: &'static str, + no_default_features: bool, + features: &'static [&'static str], + magic: [u8; 3], +} + +const VARIANTS: [Variant; 9] = [ + Variant { + name: "runimage", + description: "RunImage (SquashFS + DwarFS)", + no_default_features: false, + features: &[], + magic: *b"RI\x02", + }, + Variant { + name: "runimage-squashfs", + description: "RunImage (SquashFS-only)", + no_default_features: true, + features: &["squashfs"], + magic: *b"RI\x02", + }, + Variant { + name: "runimage-dwarfs", + description: "RunImage (DwarFS-only)", + no_default_features: true, + features: &["dwarfs"], + magic: *b"RI\x02", + }, + Variant { + name: "appimage", + description: "AppImage (SquashFS + DwarFS)", + no_default_features: false, + features: &["appimage"], + magic: *b"AI\x02", + }, + Variant { + name: "appimage-lite", + description: "AppImage lite (SquashFS + DwarFS)", + no_default_features: false, + features: &["appimage", "lite"], + magic: *b"AI\x02", + }, + Variant { + name: "appimage-squashfs", + description: "AppImage (SquashFS-only)", + no_default_features: true, + features: &["appimage", "squashfs"], + magic: *b"AI\x02", + }, + Variant { + name: "appimage-squashfs-lite", + description: "AppImage lite (SquashFS-only)", + no_default_features: true, + features: &["appimage", "squashfs", "lite"], + magic: *b"AI\x02", + }, + Variant { + name: "appimage-dwarfs", + description: "AppImage (DwarFS-only)", + no_default_features: true, + features: &["appimage", "dwarfs"], + magic: *b"AI\x02", + }, + Variant { + name: "appimage-dwarfs-lite", + description: "AppImage lite (DwarFS-only)", + no_default_features: true, + features: &["appimage", "dwarfs", "lite"], + magic: *b"AI\x02", + }, +]; + +impl Variant { + fn features_arg(self) -> Option { + (!self.features.is_empty()).then(|| self.features.join(",")) } } -fn try_main() -> Result<(), DynError> { - let all_bins = vec![ - "runimage-x86_64", - "appimage-x86_64", - "appimage-lite-x86_64", - "appimage-squashfs-x86_64", - "appimage-squashfs-lite-x86_64", - "appimage-dwarfs-x86_64", - "appimage-dwarfs-lite-x86_64", - - "runimage-aarch64", - "appimage-aarch64", - "appimage-lite-aarch64", - "appimage-squashfs-aarch64", - "appimage-squashfs-lite-aarch64", - "appimage-dwarfs-aarch64", - "appimage-dwarfs-lite-aarch64", +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +struct Task { + name: String, + arch_index: usize, + variant_index: usize, +} + +impl Task { + fn arch(&self) -> &'static Arch { + &ARCHES[self.arch_index] + } + + fn variant(&self) -> &'static Variant { + &VARIANTS[self.variant_index] + } + + fn output_name(&self) -> String { + format!("{BIN_NAME}-{}", self.name) + } +} + +const BUILD_BACKEND: &str = "cargo + Zig 0.16.0"; + +fn arch_by_name(name: &str) -> Option<&'static Arch> { + ARCHES.iter().find(|arch| arch.artifact_name == name) +} + +fn all_tasks() -> Vec { + ARCHES + .iter() + .enumerate() + .flat_map(|(arch_index, arch)| { + VARIANTS + .iter() + .enumerate() + .map(move |(variant_index, variant)| Task { + name: format!("{}-{}", variant.name, arch.artifact_name), + arch_index, + variant_index, + }) + }) + .collect() +} + +fn select_tasks(args: &[&str]) -> Result, String> { + if args.len() != 1 { + let extra = args.get(1).copied().unwrap_or(""); + return Err(format!( + "unexpected extra argument `{extra}`; pass exactly one task (UPX is not supported)" + )); + } + + let tasks = all_tasks(); + if args[0] == "all" { + Ok(tasks) + } else if arch_by_name(args[0]).is_some() { + Ok(tasks + .into_iter() + .filter(|task| task.arch().artifact_name == args[0]) + .collect()) + } else if let Some(task) = tasks.into_iter().find(|task| task.name == args[0]) { + Ok(vec![task]) + } else { + Err(format!( + "unknown task `{}`; run `cargo xtask help` to list the 54 valid tasks", + args[0] + )) + } +} + +fn cargo_build_args(arch: &Arch, variant: &Variant) -> Vec { + let mut args = vec![ + "build".into(), + "--locked".into(), + "--release".into(), + "--target".into(), + arch.rust_target.into(), ]; - let arg = env::args().nth(1).unwrap_or_else(||{ - "".into() - }); - let arg = arg.as_str(); + if variant.no_default_features { + args.push("--no-default-features".into()); + } + if let Some(features) = variant.features_arg() { + args.push("--features".into()); + args.push(features); + } + args +} - if all_bins.contains(&arg) { - build(arg)?; - return Ok(()) +fn target_is_foreign(arch: &Arch, host_arch: &str) -> bool { + arch.artifact_name != host_arch +} + +fn host_artifact_arch() -> &'static str { + match env::consts::ARCH { + "powerpc64" if cfg!(target_endian = "little") => "ppc64le", + "powerpc64" => "ppc64", + other => other, } +} - match arg { - "all" => { - for bin in all_bins { - build(bin)? - } - }, - "x86_64" => { - for bin in all_bins.iter().filter(|&bin| bin.ends_with("x86_64")) { - build(bin)? - } - }, - "aarch64" => { - for bin in all_bins.iter().filter(|&bin| bin.ends_with("aarch64")) { - build(bin)? - } - }, - _ => print_help(), +fn default_check_target(os: &str, arch: &str, little_endian: bool) -> Result<&'static str, String> { + if os != "linux" { + return Err(format!( + "cannot infer a default musl target for {arch}-{os}; pass one of the supported Rust targets explicitly" + )); + } + match arch { + "x86_64" => Ok("x86_64-unknown-linux-musl"), + "aarch64" => Ok("aarch64-unknown-linux-musl"), + "riscv64" => Ok("riscv64gc-unknown-linux-musl"), + "loongarch64" => Ok("loongarch64-unknown-linux-musl"), + "powerpc64" if little_endian => Ok("powerpc64le-unknown-linux-musl"), + "powerpc64" => Ok("powerpc64-unknown-linux-musl"), + _ => Err(format!( + "cannot infer a supported musl target for host architecture `{arch}`" + )), } - Ok(()) } -fn create_dist_dir() -> Result<(), DynError> { - create_dir_all(dist_dir())?; +fn check_commands(target: &str) -> Vec> { + vec![ + vec!["fmt".into(), "--check".into()], + vec![ + "check".into(), + "--locked".into(), + "--workspace".into(), + "--all-features".into(), + "--target".into(), + target.into(), + ], + vec![ + "clippy".into(), + "--locked".into(), + "--workspace".into(), + "--all-features".into(), + "--all-targets".into(), + "--target".into(), + target.into(), + "--".into(), + "-D".into(), + "warnings".into(), + ], + vec![ + "test".into(), + "--locked".into(), + "--workspace".into(), + "--all-features".into(), + "--target".into(), + target.into(), + ], + vec![ + "check".into(), + "--locked".into(), + "--manifest-path".into(), + "xtask/Cargo.toml".into(), + ], + vec![ + "clippy".into(), + "--locked".into(), + "--manifest-path".into(), + "xtask/Cargo.toml".into(), + "--all-targets".into(), + "--".into(), + "-D".into(), + "warnings".into(), + ], + vec![ + "test".into(), + "--locked".into(), + "--manifest-path".into(), + "xtask/Cargo.toml".into(), + ], + ] +} + +fn run_status(program: &str, args: &[String]) -> Result<(), DynError> { + eprintln!("+ {program} {}", args.join(" ")); + let status = Command::new(program) + .current_dir(project_root()) + .args(args) + .status()?; + if status.success() { + Ok(()) + } else { + Err(format!("`{program} {}` failed with {status}", args.join(" ")).into()) + } +} + +fn qemu_runner_names(arch: &Arch) -> [String; 2] { + let name = match arch.artifact_name { + "x86_64" => "qemu-x86_64", + "aarch64" => "qemu-aarch64", + "riscv64" => "qemu-riscv64", + "loongarch64" => "qemu-loongarch64", + "ppc64" => "qemu-ppc64", + "ppc64le" => "qemu-ppc64le", + _ => unreachable!("all check targets come from ARCHES"), + }; + [name.to_string(), format!("{name}-static")] +} + +fn configure_target_runner(command: &mut Command, arch: &Arch) -> Result<(), DynError> { + let project = project_root(); + let names = qemu_runner_names(arch); + let runner = names + .iter() + .find_map(|name| resolve_program(Path::new(name), &project).ok()) + .ok_or_else(|| { + format!( + "foreign tests for {} require `{}` or `{}` in PATH", + arch.rust_target, names[0], names[1] + ) + })?; + command.env( + target_env_key("CARGO_TARGET", arch.rust_target, "_RUNNER"), + runner, + ); Ok(()) } -fn print_help() { - eprintln!("Tasks: - x86_64 build x86_64 RunImage and AppImage uruntime - runimage-x86_64 build x86_64 RunImage uruntime - runimage-squashfs-x86_64 build x86_64 RunImage uruntime (SquashFS-only) - runimage-dwarfs-x86_64 build x86_64 RunImage uruntime (DwarFS-only) - appimage-x86_64 build x86_64 AppImage uruntime - appimage-lite-x86_64 build x86_64 AppImage uruntime (no dwarfsck, mkdwarfs, mksquashfs, sqfstar) - appimage-squashfs-x86_64 build x86_64 AppImage uruntime (SquashFS-only) - appimage-squashfs-lite-x86_64 build x86_64 AppImage uruntime (SquashFS-only no mksquashfs, sqfstar) - appimage-dwarfs-x86_64 build x86_64 AppImage uruntime (DwarFS-only) - appimage-dwarfs-lite-x86_64 build x86_64 AppImage uruntime (DwarFS-only no dwarfsck, mkdwarfs) - - aarch64 build aarch64 RunImage and AppImage uruntime - runimage-aarch64 build aarch64 RunImage uruntime - runimage-squashfs-aarch64 build aarch64 RunImage uruntime (SquashFS-only) - runimage-dwarfs-aarch64 build aarch64 RunImage uruntime (DwarFS-only) - appimage-aarch64 build aarch64 AppImage uruntime - appimage-lite-aarch64 build aarch64 AppImage uruntime (no dwarfsck, mkdwarfs, mksquashfs, sqfstar) - appimage-squashfs-aarch64 build aarch64 AppImage uruntime (SquashFS-only) - appimage-squashfs-lite-aarch64 build aarch64 AppImage uruntime (SquashFS-only no mksquashfs, sqfstar) - appimage-dwarfs-aarch64 build aarch64 AppImage uruntime (DwarFS-only) - appimage-dwarfs-lite-aarch64 build aarch64 AppImage uruntime (DwarFS-only no dwarfsck, mkdwarfs) - - all build all of the above") -} - -fn strip(path: &PathBuf) -> Result<(), DynError> { - if Command::new("strip") - .arg("--version") - .stdout(Stdio::null()) - .status() - .is_ok() - { - eprint!(" stripping: "); - let status = Command::new("strip").args([ - "-s", "-R", ".comment", "-R", ".gnu.version", - "--strip-unneeded" - ]).arg(path).status()?; - if !status.success() { - Err("strip failed")?; +fn check_command_requirements(args: &[String], foreign: bool) -> (bool, bool) { + let targets_root_package = args.iter().any(|arg| arg == "--target"); + let uses_zig = targets_root_package; + let uses_runner = foreign + && targets_root_package + && args.first().is_some_and(|arg| arg == "test"); + (uses_zig, uses_runner) +} + +fn run_check_command(args: &[String], arch: &Arch, foreign: bool) -> Result<(), DynError> { + eprintln!("+ cargo {}", args.join(" ")); + let mut command = Command::new("cargo"); + command.current_dir(project_root()).args(args); + let (uses_zig, uses_runner) = check_command_requirements(args, foreign); + if uses_zig { + configure_zig(&mut command, arch)?; + if uses_runner { + configure_target_runner(&mut command, arch)?; } - eprint!("OK"); + } + let status = command.status()?; + if status.success() { + Ok(()) } else { - Err("no `strip` utility found!")?; + Err(format!("`cargo {}` failed with {status}", args.join(" ")).into()) } +} + +fn run_checks(target: &str) -> Result<(), DynError> { + let arch = ARCHES + .iter() + .find(|arch| arch.rust_target == target) + .ok_or_else(|| { + format!( + "unsupported check target `{target}`; use one of the Rust targets shown by `cargo xtask help`" + ) + })?; + let foreign = target_is_foreign(arch, host_artifact_arch()); + eprintln!( + "running local checks for Rust target {target}, backend={}", + BUILD_BACKEND + ); + for args in check_commands(target) { + run_check_command(&args, arch, foreign)?; + } + eprintln!("+ cargo xtask update-checksums --check"); + update_checksums(true)?; + run_status("git", &["diff".into(), "--check".into()])?; + eprintln!("all local checks passed for {target}"); Ok(()) } -fn add_sections(path: &PathBuf) -> Result<(), DynError> { - if Command::new("llvm-objcopy") - .arg("--version") - .stdout(Stdio::null()) - .status() - .is_ok() - { - eprint!(" add sections: "); - - if let Ok(dir) = sections_dir().read_dir() { - let mut objcopy_args = Vec::new(); - for entry in dir.flatten() { - let section_file = entry.path(); - if section_file.is_file() { - let section_name = format!(".{}", section_file.file_name().unwrap_or_default().to_string_lossy()); - objcopy_args.append(&mut vec![ - format!("--add-section={section_name}={}", section_file.display()), - format!("--set-section-flags={section_name}=noload,readonly"), - ]); - } - } - let status = Command::new("llvm-objcopy") - .args(objcopy_args).arg(path).status()?; - if !status.success() { - Err("failed to add sections")?; - } +fn resolve_program(program: &Path, base: &Path) -> Result { + let has_separator = program.components().count() > 1; + if program.is_absolute() || has_separator { + let candidate = if program.is_absolute() { + program.to_path_buf() + } else { + base.join(program) + }; + if candidate.is_file() { + return Ok(candidate); } - eprint!("OK"); - } else { - Err("no `llvm-objcopy` utility found!")?; + return Err(format!("program not found: {}", candidate.display()).into()); + } + + let path = env::var_os("PATH").ok_or("PATH is not set")?; + for directory in env::split_paths(&path) { + let directory = if directory.is_absolute() { + directory + } else { + base.join(directory) + }; + let candidate = directory.join(program); + if candidate.is_file() { + return Ok(candidate); + } + } + Err(format!("program `{}` was not found in PATH", program.display()).into()) +} + +fn require_tool(path: &Path, display_name: &str) -> Result<(), DynError> { + if path.is_file() { + return Ok(()); + } + Err(format!("required {display_name} not found at {}", path.display()).into()) +} + +fn zig_version(path: &Path) -> Result { + let output = Command::new(path).arg("version").output()?; + if !output.status.success() { + return Err(format!("failed to execute `{}` version", path.display()).into()); + } + Ok(std::str::from_utf8(&output.stdout)?.trim().to_string()) +} + +fn require_zig_version(path: &Path) -> Result<(), DynError> { + let version = zig_version(path)?; + if version != ZIG_VERSION { + return Err(format!( + "Zig {ZIG_VERSION} is required for reproducible foreign builds, but `{}` reported `{version}`", + path.display() + ) + .into()); } Ok(()) } -fn add_magic(path: &PathBuf, magic: &str) -> Result<(), DynError> { - eprint!(" embed magic: "); - let magic = format!("{}\x02", magic); - let mut file = OpenOptions::new() +fn zig_archive_path(toolchains: &Path, package: &ZigPackage) -> PathBuf { + toolchains.join(format!( + "zig-{}-{}-{}.tar.xz", + package.version, package.platform, package.sha256 + )) +} + +fn legacy_zig_archive_path(toolchains: &Path, package: &ZigPackage) -> PathBuf { + toolchains.join(format!( + "zig-{}-{}.tar.xz", + package.version, package.platform + )) +} + +fn download_zig_archive( + curl: &OsStr, + package: &ZigPackage, + archive_path: &Path, +) -> Result<(), DynError> { + let toolchains = archive_path + .parent() + .ok_or("Zig archive path has no parent")?; + create_dir_all(toolchains)?; + let archive = tempfile::NamedTempFile::new_in(toolchains)?; + let archive_output = archive.reopen()?; + let status = Command::new(curl) + .args(["--fail", "--location", "--retry", "3", "--max-filesize"]) + .arg(ZIG_DOWNLOAD_MAX.to_string()) + .arg(&package.url) + .stdout(Stdio::from(archive_output)) + .status()?; + if !status.success() { + return Err(format!( + "failed to download pinned Zig {} from {}", + package.version, package.url + ) + .into()); + } + archive.as_file().sync_all()?; + let actual = build_support::sha256_file(archive.path(), ZIG_DOWNLOAD_MAX) + .map_err(|error| -> DynError { error.into() })?; + if actual != package.sha256 { + return Err(format!( + "SHA-256 mismatch for Zig {}: expected {}, got {actual}", + package.version, package.sha256 + ) + .into()); + } + archive.persist(archive_path)?; + File::open(toolchains)?.sync_all()?; + Ok(()) +} + +fn ensure_zig_archive( + curl: &OsStr, + toolchains: &Path, + package: &ZigPackage, +) -> Result { + let archive_path = zig_archive_path(toolchains, package); + with_cache_lock(&archive_path, || { + let actual = build_support::sha256_file(&archive_path, ZIG_DOWNLOAD_MAX); + if actual.as_deref() == Ok(package.sha256.as_str()) { + eprintln!("using verified cached {}", archive_path.display()); + return Ok(archive_path.clone()); + } + let legacy_path = legacy_zig_archive_path(toolchains, package); + let legacy_actual = build_support::sha256_file(&legacy_path, ZIG_DOWNLOAD_MAX); + if legacy_actual.as_deref() == Ok(package.sha256.as_str()) { + fs::rename(&legacy_path, &archive_path).map_err(|error| { + format!( + "failed to migrate {} to {}: {error}", + legacy_path.display(), + archive_path.display() + ) + })?; + eprintln!("using verified cached {}", archive_path.display()); + return Ok(archive_path.clone()); + } + eprintln!( + "downloading pinned Zig {} from {}", + package.version, package.url + ); + download_zig_archive(curl, package, &archive_path).map_err(|error| error.to_string())?; + Ok(archive_path.clone()) + }) + .map_err(|error| error.into()) +} + +fn ensure_zig() -> Result { + if let Some(override_path) = env::var_os("URUNTIME_ZIG") { + let path = resolve_program(&PathBuf::from(override_path), &env::current_dir()?)?; + require_zig_version(&path)?; + return Ok(path); + } + + let package = zig_package(env::consts::OS, env::consts::ARCH)?; + let toolchains = project_root().join("target/toolchains"); + create_dir_all(&toolchains)?; + let install = toolchains.join(format!( + "zig-{ZIG_VERSION}-{}-{}", + package.platform, package.sha256 + )); + let zig = install.join("zig"); + if zig.is_file() && require_zig_version(&zig).is_ok() { + return Ok(zig); + } + + let lock_path = toolchains.join(format!( + ".zig-{ZIG_VERSION}-{}-{}.lock", + package.platform, package.sha256 + )); + let lock = OpenOptions::new() + .read(true) .write(true) .create(true) .truncate(false) - .open(path)?; - file.seek(SeekFrom::Start(8))?; - file.write_all(magic[..3].as_bytes())?; - eprint!("OK"); + .open(&lock_path)?; + lock.lock_exclusive()?; + if zig.is_file() && require_zig_version(&zig).is_ok() { + return Ok(zig); + } + + let archive_path = ensure_zig_archive(OsStr::new("curl"), &toolchains, &package)?; + + let stage = tempfile::Builder::new() + .prefix(".zig-install-") + .tempdir_in(&toolchains)?; + let status = Command::new("tar") + .args(["-xJf"]) + .arg(&archive_path) + .arg("-C") + .arg(stage.path()) + .arg("--strip-components=1") + .status()?; + if !status.success() { + return Err(format!("failed to extract pinned Zig {ZIG_VERSION}").into()); + } + let staged_zig = stage.path().join("zig"); + require_zig_version(&staged_zig)?; + if install.exists() { + fs::remove_dir_all(&install)?; + } + fs::rename(stage.keep(), &install)?; + File::open(&toolchains)?.sync_all()?; + eprintln!("installed Zig {ZIG_VERSION} at {}", zig.display()); + Ok(zig) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ManifestUpdate { + Updated, + Unchanged, +} + +fn render_helper_section_with( + sources: &[build_support::AssetSource], + fetch: &F, +) -> Result +where + F: Fn(&build_support::AssetSource) -> Result, String>, +{ + let mut rows = Vec::with_capacity(sources.len()); + for source in sources { + eprintln!( + "validating {}/{} from {}", + source.target.release_arch, source.name, source.url + ); + let bytes = fetch(source).map_err(|error| -> DynError { error.into() })?; + if bytes.len() > build_support::MAX_DOWNLOAD_SIZE { + return Err(format!( + "downloaded {}/{} is {} bytes, exceeding limit {}", + source.target.release_arch, + source.name, + bytes.len(), + build_support::MAX_DOWNLOAD_SIZE + ) + .into()); + } + let source_sha256 = build_support::sha256_hex(&bytes); + let payload = match source.kind { + build_support::AssetKind::Direct => bytes, + build_support::AssetKind::DwarfsWrapper => build_support::extract_dwarfs_wrapper( + &bytes, + source.target, + build_support::MAX_HELPER_SIZE, + ) + .map_err(|error| -> DynError { error.into() })?, + }; + build_support::validate_elf(&payload, source.target, source.elf_type) + .map_err(|error| -> DynError { error.into() })?; + rows.push(( + source.target.release_arch, + source.name, + source_sha256, + build_support::sha256_hex(&payload), + )); + } + rows.sort_by(|left, right| (left.0, left.1).cmp(&(right.0, right.1))); + if rows + .windows(2) + .any(|pair| (pair[0].0, pair[0].1) == (pair[1].0, pair[1].1)) + { + return Err("duplicate helper source inventory entry".into()); + } + let mut section = String::from("[helpers]\n# arch\tname\tsource_sha256\tpayload_sha256\n"); + for (arch, name, source_sha256, payload_sha256) in rows { + section.push_str(&format!( + "{arch}\t{name}\t{source_sha256}\t{payload_sha256}\n" + )); + } + Ok(section) +} + +fn render_zig_manifest_section(packages: &[ZigPackage]) -> String { + let mut section = String::from("[zig]\n# version\tplatform\turl\tsha256\n"); + for package in packages { + section.push_str(&format!( + "{}\t{}\t{}\t{}\n", + package.version, package.platform, package.url, package.sha256 + )); + } + section +} + +fn cached_zig_index_matches_manifest(index: &[u8]) -> bool { + let Ok(packages) = zig_packages() else { + return false; + }; + render_zig_section(index).is_ok_and(|section| section == render_zig_manifest_section(&packages)) +} + +fn zig_package_from_index(index: &[u8], os: &str, arch: &str) -> Result { + let platform = zig_platform(os, arch) + .ok_or_else(|| format!("unsupported Zig host platform {arch}-{os}"))?; + let section = render_zig_section(index)?; + parse_zig_packages(§ion)? + .into_iter() + .find(|package| package.platform == platform) + .ok_or_else(|| format!("Zig {ZIG_VERSION} index has no {platform} package").into()) +} + +fn render_zig_section(index: &[u8]) -> Result { + let root: serde_json::Value = serde_json::from_slice(index)?; + let release = root + .get(ZIG_VERSION) + .and_then(serde_json::Value::as_object) + .ok_or_else(|| format!("Zig {ZIG_VERSION} is absent from {ZIG_INDEX_URL}"))?; + let mut section = String::from("[zig]\n# version\tplatform\turl\tsha256\n"); + for platform in ZIG_PLATFORMS { + let package = release + .get(platform) + .and_then(serde_json::Value::as_object) + .ok_or_else(|| format!("Zig {ZIG_VERSION} has no {platform} package"))?; + let url = package + .get("tarball") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| format!("Zig {ZIG_VERSION} {platform} has no tarball URL"))?; + let sha256 = package + .get("shasum") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| format!("Zig {ZIG_VERSION} {platform} has no SHA-256"))?; + let expected_url = zig_download_url(platform); + if url != expected_url || !valid_sha256(sha256) { + return Err(format!("invalid Zig {ZIG_VERSION} metadata for {platform}").into()); + } + section.push_str(&format!("{ZIG_VERSION}\t{platform}\t{url}\t{sha256}\n")); + } + Ok(section) +} + +fn render_checksum_manifest_with( + sources: &[build_support::AssetSource], + fetch: &F, + zig_index: &[u8], +) -> Result +where + F: Fn(&build_support::AssetSource) -> Result, String>, +{ + Ok(format!( + "# uruntime checksum manifest v1\n\n{}\n{}", + render_helper_section_with(sources, fetch)?, + render_zig_section(zig_index)? + )) +} + +fn update_checksum_manifest_with( + manifest_path: &Path, + check: bool, + sources: &[build_support::AssetSource], + fetch: &F, + zig_index: &[u8], +) -> Result +where + F: Fn(&build_support::AssetSource) -> Result, String>, +{ + let generated = render_checksum_manifest_with(sources, fetch, zig_index)?; + let current = match fs::read(manifest_path) { + Ok(current) => current, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(error) => { + return Err(format!( + "failed to read checksum manifest {}: {error}", + manifest_path.display() + ) + .into()) + } + }; + if current == generated.as_bytes() { + return Ok(ManifestUpdate::Unchanged); + } + if check { + return Err(format!( + "checksum manifest drift detected at {}; rerun `cargo xtask update-checksums` and review the diff", + manifest_path.display() + ) + .into()); + } + build_support::atomic_write(manifest_path, generated.as_bytes()) + .map_err(|error| -> DynError { error.into() })?; + Ok(ManifestUpdate::Updated) +} + +fn helper_source_cache_path( + project: &Path, + source: &build_support::AssetSource, +) -> Result { + Ok(project.join(source_cache_relative_path( + source.target, + source.name, + source.kind, + )?)) +} + +fn read_cached_helper_source(path: &Path) -> Result>, String> { + let metadata = match fs::metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(format!("failed to inspect {}: {error}", path.display())), + }; + if !metadata.is_file() { + return Err(format!("{} is not a regular file", path.display())); + } + if metadata.len() > MAX_DOWNLOAD_SIZE as u64 { + return Err(format!( + "{} size {} exceeds limit {MAX_DOWNLOAD_SIZE}", + path.display(), + metadata.len() + )); + } + fs::read(path) + .map(Some) + .map_err(|error| format!("failed to read {}: {error}", path.display())) +} + +fn read_verified_cached_helper_source( + path: &Path, + expected_sha256: Option<&str>, +) -> Result>, String> { + let Some(bytes) = read_cached_helper_source(path)? else { + return Ok(None); + }; + let actual = build_support::sha256_hex(&bytes); + Ok((expected_sha256 == Some(actual.as_str())).then_some(bytes)) +} + +fn update_checksums(check: bool) -> Result<(), DynError> { + let curl = env::var_os("URUNTIME_CURL").unwrap_or_else(|| "curl".into()); + let project = project_root(); + let current_digests = build_support::digest_records() + .map_err(|error| format!("failed to read current helper checksums: {error}"))?; + let fetch = |source: &build_support::AssetSource| -> Result, String> { + let destination = helper_source_cache_path(&project, source)?; + let expected = current_digests + .iter() + .find(|record| { + record.arch == source.target.release_arch && record.name == source.name + }) + .map(|record| record.source_sha256.as_str()); + with_cache_lock(&destination, || { + if let Some(bytes) = read_verified_cached_helper_source(&destination, expected)? { + eprintln!("using verified cached {}", destination.display()); + return Ok(bytes); + } + if destination.exists() { + eprintln!( + "cached {} does not match the current manifest; downloading it again", + destination.display() + ); + } else { + eprintln!("cache miss for {}; downloading it", destination.display()); + } + build_support::download_atomic(&curl, &source.url, &destination)?; + read_cached_helper_source(&destination)?.ok_or_else(|| { + format!( + "downloaded helper disappeared from {}", + destination.display() + ) + }) + }) + }; + let toolchains = project.join("target/toolchains"); + let zig_index_path = toolchains.join("zig-index.json"); + let zig_index = with_cache_lock(&zig_index_path, || { + if let Some(index) = read_cached_helper_source(&zig_index_path)? { + if cached_zig_index_matches_manifest(&index) { + eprintln!("using verified cached {}", zig_index_path.display()); + return Ok(index); + } + eprintln!( + "cached {} does not match the current Zig manifest; downloading it again", + zig_index_path.display() + ); + } else { + eprintln!("cache miss for {}; downloading it", zig_index_path.display()); + } + build_support::download_atomic(&curl, ZIG_INDEX_URL, &zig_index_path)?; + read_cached_helper_source(&zig_index_path)?.ok_or_else(|| { + format!( + "downloaded Zig index disappeared from {}", + zig_index_path.display() + ) + }) + }) + .map_err(|error| -> DynError { error.into() })?; + eprintln!( + "validating Zig {ZIG_VERSION} metadata for {} supported Linux hosts", + ZIG_PLATFORMS.len() + ); + let host_zig = zig_package_from_index(&zig_index, env::consts::OS, env::consts::ARCH)?; + ensure_zig_archive(curl.as_os_str(), &toolchains, &host_zig)?; + let manifest_path = project.join("checksums.txt"); + match update_checksum_manifest_with( + &manifest_path, + check, + &build_support::all_asset_sources(), + &fetch, + &zig_index, + )? { + ManifestUpdate::Updated => eprintln!("updated {}", manifest_path.display()), + ManifestUpdate::Unchanged => eprintln!("{} is current", manifest_path.display()), + } Ok(()) } -fn build(bin: &str) -> Result<(), DynError> { - create_dist_dir()?; +fn main() { + if let Err(error) = try_main() { + eprintln!("error: {error}"); + exit(1); + } +} - let cargo: &str; - let target: &str; - let mut is_strip = true; - let mut build_args = Vec::new(); - - if bin.ends_with("aarch64") { - cargo = "cross"; - is_strip = false; - target = TARGET_AARCH64; - build_args.append(&mut vec![ - "build", "--release", - "--target", target - ]) - } else { - cargo = "cargo"; - target = TARGET_X86_64; - build_args.append(&mut vec![ - "+nightly", "build", "--release", - "--target", target, - "-Z", "unstable-options", "-Z", "build-std=std,panic_abort", - "-Z", "build-std-features=panic_immediate_abort" - ]) +fn try_main() -> Result<(), DynError> { + let args: Vec = env::args().skip(1).collect(); + if args.is_empty() || (args.len() == 1 && matches!(args[0].as_str(), "help" | "-h" | "--help")) + { + eprint!("{}", help_text()); + return Ok(()); } - if bin.contains("squashfs") { - build_args.append(&mut vec!["--no-default-features", "--features", "squashfs"]); - } else if bin.contains("dwarfs") { - build_args.append(&mut vec!["--no-default-features", "--features", "dwarfs"]); + if args.first().map(String::as_str) == Some("check") { + let target = match args.as_slice() { + [_] => default_check_target( + env::consts::OS, + env::consts::ARCH, + cfg!(target_endian = "little"), + ) + .map_err(|error| -> DynError { error.into() })?, + [_, target] => ARCHES + .iter() + .find(|arch| arch.rust_target == target) + .map(|arch| arch.rust_target) + .ok_or_else(|| format!("unsupported check target `{target}`"))?, + _ => return Err("usage: cargo xtask check [RUST_TARGET]".into()), + }; + return run_checks(target); } - let mut magic = "RI"; - if bin.contains("appimage") { - magic = "AI"; - build_args.append(&mut vec!["--features", "appimage"]) + + if args.first().map(String::as_str) == Some("update-checksums") { + return match args.as_slice() { + [_] => update_checksums(false), + [_, flag] if flag == "--check" => update_checksums(true), + _ => Err("usage: cargo xtask update-checksums [--check]" + .to_string() + .into()), + }; } - if bin.contains("lite") { - build_args.append(&mut vec!["--features", "lite"]); + + let arg_refs: Vec<&str> = args.iter().map(String::as_str).collect(); + for task in select_tasks(&arg_refs).map_err(|error| -> DynError { error.into() })? { + build(&task)?; } + Ok(()) +} - let upx = env::args().nth(2).unwrap_or_default().to_lowercase() == "--upx"; - if upx { build_args.append(&mut vec!["--features", "upx"]) } +fn help_text() -> String { + let mut text = String::from( + "Usage:\n cargo xtask \n cargo xtask check [RUST_TARGET]\n cargo xtask update-checksums [--check]\n\nArchitectures:\n", + ); + for arch in ARCHES { + text.push_str(&format!( + " {:12} {} ({:?}-endian)\n", + arch.artifact_name, arch.rust_target, arch.endian + )); + } + text.push_str(&format!("\nTasks ({}):\n", ARCHES.len() * VARIANTS.len())); + for task in all_tasks() { + text.push_str(&format!( + " {:38} {}\n", + task.name, + task.variant().description + )); + } + text.push_str("\n all build all 54 tasks\n"); + text +} - let status = Command::new(cargo) - .current_dir(project_root()) - .args(build_args) - .status()?; +fn create_dist_dir() -> Result<(), DynError> { + create_dir_all(dist_dir())?; + Ok(()) +} +fn add_sections(path: &Path, section_root: &Path, objcopy: &OsStr) -> Result<(), DynError> { + let mut section_files = section_root.read_dir()?.collect::, _>>()?; + section_files.sort_by_key(|entry| entry.file_name()); + + let mut args = Vec::::new(); + for entry in section_files { + let section_file = entry.path(); + if !section_file.is_file() { + continue; + } + let filename = section_file + .file_name() + .and_then(OsStr::to_str) + .ok_or("section filenames must be valid UTF-8")?; + let section_path = section_file + .to_str() + .ok_or("section paths must be valid UTF-8 for llvm-objcopy")?; + let section_name = format!(".{filename}"); + args.push(format!("--add-section={section_name}={section_path}").into()); + args.push(format!("--set-section-flags={section_name}=noload,readonly").into()); + } + args.push(path.as_os_str().to_owned()); + + let status = Command::new(objcopy).args(args).status()?; if !status.success() { - Err("cargo build failed")?; + return Err("llvm-objcopy failed to add runtime sections".into()); } + Ok(()) +} - let src = project_root() - .join("target") - .join(target) - .join("release") - .join(BIN_NAME); +fn add_magic(path: &Path, magic: [u8; 3]) -> Result<(), DynError> { + let mut file = OpenOptions::new().write(true).open(path)?; + file.seek(SeekFrom::Start(8))?; + file.write_all(&magic)?; + file.sync_all()?; + Ok(()) +} - let dst_bin_name = if upx { - &format!("{BIN_NAME}-{bin}-upx") - } else { - &format!("{BIN_NAME}-{bin}") - }; - let dst = dist_dir().join(dst_bin_name); +fn publish_artifact( + src: &Path, + dst: &Path, + section_root: &Path, + magic: [u8; 3], +) -> Result<(), DynError> { + let parent = dst.parent().ok_or("artifact output has no parent")?; + let mut temporary = tempfile::NamedTempFile::new_in(parent)?; + let mut source = File::open(src)?; + std::io::copy(&mut source, temporary.as_file_mut())?; + temporary + .as_file_mut() + .set_permissions(source.metadata()?.permissions())?; + temporary.as_file_mut().sync_all()?; + add_sections(temporary.path(), section_root, OsStr::new("llvm-objcopy"))?; + add_magic(temporary.path(), magic)?; + temporary.as_file().sync_all()?; + temporary.persist(dst)?; + File::open(parent)?.sync_all()?; + Ok(()) +} - rename(&src, &dst)?; - eprint!("{dst_bin_name}: OK"); +fn target_env_key(prefix: &str, target: &str, suffix: &str) -> String { + format!( + "{prefix}_{}{suffix}", + target.replace('-', "_").to_ascii_uppercase() + ) +} - if is_strip { - strip(&dst)?; +fn query_rust_sysroot(project: &Path, compiler: &OsStr) -> Result { + let compiler = resolve_program(Path::new(compiler), project)?; + let output = Command::new(&compiler) + .current_dir(project) + .args(["--print", "sysroot"]) + .output()?; + if !output.status.success() { + return Err(format!( + "failed to determine the active Rust sysroot with {}", + compiler.display() + ) + .into()); } + Ok(std::str::from_utf8(&output.stdout)?.trim().to_string()) +} - add_sections(&dst)?; +fn configure_zig(command: &mut Command, arch: &Arch) -> Result<(), DynError> { + let zig = ensure_zig()?; + let project = project_root(); + let wrapper = project.join("scripts/zig-linker.sh"); + require_tool(&wrapper, "project Zig linker wrapper")?; + let rustc = env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let rust_sysroot = query_rust_sysroot(&project, &rustc)?; + command + .env( + target_env_key("CARGO_TARGET", arch.rust_target, "_LINKER"), + &wrapper, + ) + .env( + target_env_key("CARGO_TARGET", arch.rust_target, "_RUSTFLAGS"), + "-Ctarget-feature=+crt-static -Clink-self-contained=no", + ) + .env( + format!("CC_{}", arch.rust_target.replace('-', "_")), + &wrapper, + ) + .env("URUNTIME_RUST_TARGET", arch.rust_target) + .env("URUNTIME_ZIG_TARGET", arch.zig_target) + .env("URUNTIME_RUST_SYSROOT", rust_sysroot) + .env("URUNTIME_ZIG", zig); + Ok(()) +} - add_magic(&dst, magic)?; +fn build(task: &Task) -> Result<(), DynError> { + create_dist_dir()?; + eprintln!( + "building {}: artifact arch={}, Rust target={}, backend={}", + task.name, + task.arch().artifact_name, + task.arch().rust_target, + BUILD_BACKEND + ); + + let mut command = Command::new("cargo"); + command + .current_dir(project_root()) + .args(cargo_build_args(task.arch(), task.variant())); + configure_zig(&mut command, task.arch())?; + let status = command.status()?; + if !status.success() { + return Err(format!("cargo build failed for {}", task.name).into()); + } - eprintln!(); + let src = project_root() + .join("target") + .join(task.arch().rust_target) + .join("release") + .join(BIN_NAME); + let dst_name = task.output_name(); + let dst = dist_dir().join(&dst_name); + publish_artifact(&src, &dst, §ions_dir(), task.variant().magic)?; + eprintln!("{dst_name}: OK"); Ok(()) } fn project_root() -> PathBuf { - Path::new(&env!("CARGO_MANIFEST_DIR")) - .ancestors() - .nth(1) - .unwrap() + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask must be directly below the project root") .to_path_buf() } @@ -267,3 +1263,6 @@ fn dist_dir() -> PathBuf { fn sections_dir() -> PathBuf { project_root().join("sections") } + +#[cfg(test)] +mod tests; diff --git a/xtask/src/tests.rs b/xtask/src/tests.rs new file mode 100644 index 0000000..87fde85 --- /dev/null +++ b/xtask/src/tests.rs @@ -0,0 +1,638 @@ +use super::*; +use std::collections::HashSet; + +#[test] +fn declarative_matrix_has_six_arches_and_54_unique_tasks() { + assert_eq!(ARCHES.len(), 6); + assert_eq!(VARIANTS.len(), 9); + let tasks = all_tasks(); + assert_eq!(tasks.len(), 54); + assert_eq!(tasks.iter().collect::>().len(), 54); +} + +#[test] +fn architecture_contract_includes_correct_powerpc_endian_targets() { + assert_eq!( + arch_by_name("x86_64").unwrap().rust_target, + "x86_64-unknown-linux-musl" + ); + assert_eq!( + arch_by_name("riscv64").unwrap().rust_target, + "riscv64gc-unknown-linux-musl" + ); + assert_eq!( + arch_by_name("ppc64").unwrap().rust_target, + "powerpc64-unknown-linux-musl" + ); + assert_eq!(arch_by_name("ppc64").unwrap().endian, Endian::Big); + assert_eq!( + arch_by_name("ppc64le").unwrap().rust_target, + "powerpc64le-unknown-linux-musl" + ); + assert_eq!(arch_by_name("ppc64le").unwrap().endian, Endian::Little); +} + +#[test] +fn matrix_contains_runimage_filesystem_only_tasks_and_public_output_names() { + let tasks = all_tasks(); + assert!(tasks + .iter() + .any(|task| task.name == "runimage-squashfs-loongarch64")); + assert!(tasks + .iter() + .any(|task| task.name == "runimage-dwarfs-ppc64")); + assert!(tasks + .iter() + .any(|task| task.output_name() == "uruntime-appimage-dwarfs-lite-ppc64le")); +} + +#[test] +fn variants_produce_exact_single_feature_argument() { + let cases = [ + ("runimage", false, None), + ("runimage-squashfs", true, Some("squashfs")), + ("runimage-dwarfs", true, Some("dwarfs")), + ("appimage", false, Some("appimage")), + ("appimage-lite", false, Some("appimage,lite")), + ("appimage-squashfs", true, Some("appimage,squashfs")), + ( + "appimage-squashfs-lite", + true, + Some("appimage,squashfs,lite"), + ), + ("appimage-dwarfs", true, Some("appimage,dwarfs")), + ("appimage-dwarfs-lite", true, Some("appimage,dwarfs,lite")), + ]; + for (name, no_default, features) in cases { + let variant = VARIANTS + .iter() + .find(|variant| variant.name == name) + .unwrap(); + assert_eq!(variant.no_default_features, no_default, "{name}"); + assert_eq!(variant.features_arg().as_deref(), features, "{name}"); + let args = cargo_build_args(arch_by_name("x86_64").unwrap(), variant); + assert!(args.iter().any(|arg| arg == "--locked"), "{name}"); + assert_eq!( + args.iter() + .filter(|arg| arg.as_str() == "--features") + .count(), + usize::from(features.is_some()), + "{name}" + ); + } +} + +#[test] +fn cli_selects_exact_arch_and_all_and_rejects_unknown_or_extra_args() { + assert_eq!( + select_tasks(&["appimage-squashfs-riscv64"]).unwrap().len(), + 1 + ); + assert_eq!(select_tasks(&["riscv64"]).unwrap().len(), 9); + assert_eq!(select_tasks(&["all"]).unwrap().len(), 54); + let unknown = select_tasks(&["not-a-task"]).unwrap_err(); + assert!(unknown.contains("unknown task `not-a-task`")); + assert!(unknown.contains("cargo xtask help")); + let extra = select_tasks(&["x86_64", "--upx"]).unwrap_err(); + assert!(extra.contains("unexpected extra argument `--upx`")); + assert!(extra.contains("UPX is not supported")); +} + +#[test] +fn check_command_defaults_to_the_current_platform_musl_target() { + assert_eq!( + default_check_target("linux", "x86_64", false).unwrap(), + "x86_64-unknown-linux-musl" + ); + assert_eq!( + default_check_target("linux", "aarch64", false).unwrap(), + "aarch64-unknown-linux-musl" + ); + assert_eq!( + default_check_target("linux", "powerpc64", false).unwrap(), + "powerpc64-unknown-linux-musl" + ); + assert_eq!( + default_check_target("linux", "powerpc64", true).unwrap(), + "powerpc64le-unknown-linux-musl" + ); + assert!(default_check_target("windows", "x86_64", false).is_err()); +} + +#[test] +fn check_command_contains_all_local_quality_gates() { + let commands = check_commands("x86_64-unknown-linux-musl"); + assert_eq!(commands.len(), 7); + let rendered = commands + .iter() + .map(|command| command.join(" ")) + .collect::>() + .join("\n"); + for required in [ + "fmt --check", + "check --locked --workspace --all-features --target x86_64-unknown-linux-musl", + "clippy --locked --workspace --all-features --all-targets --target x86_64-unknown-linux-musl -- -D warnings", + "test --locked --workspace --all-features --target x86_64-unknown-linux-musl", + "check --locked --manifest-path xtask/Cargo.toml", + "clippy --locked --manifest-path xtask/Cargo.toml --all-targets -- -D warnings", + "test --locked --manifest-path xtask/Cargo.toml", + ] { + assert!(rendered.contains(required), "missing `{required}` in:\n{rendered}"); + } +} + +#[test] +fn foreign_check_runners_match_every_supported_architecture() { + for (target, expected) in [ + ("aarch64-unknown-linux-musl", "qemu-aarch64"), + ("riscv64gc-unknown-linux-musl", "qemu-riscv64"), + ("loongarch64-unknown-linux-musl", "qemu-loongarch64"), + ("powerpc64-unknown-linux-musl", "qemu-ppc64"), + ("powerpc64le-unknown-linux-musl", "qemu-ppc64le"), + ] { + let arch = ARCHES.iter().find(|arch| arch.rust_target == target).unwrap(); + assert_eq!(qemu_runner_names(arch)[0], expected); + } +} + +#[test] +fn root_checks_always_use_zig_but_native_tests_do_not_use_qemu() { + let commands = check_commands("aarch64-unknown-linux-musl"); + let check = commands + .iter() + .find(|args| { + args.first().is_some_and(|arg| arg == "check") + && args.iter().any(|arg| arg == "--target") + }) + .unwrap(); + let test = commands + .iter() + .find(|args| { + args.first().is_some_and(|arg| arg == "test") + && args.iter().any(|arg| arg == "--target") + }) + .unwrap(); + let xtask_check = commands + .iter() + .find(|args| { + args.first().is_some_and(|arg| arg == "check") + && args.iter().any(|arg| arg == "xtask/Cargo.toml") + }) + .unwrap(); + + assert_eq!(check_command_requirements(check, false), (true, false)); + assert_eq!(check_command_requirements(test, false), (true, false)); + assert_eq!(check_command_requirements(test, true), (true, true)); + assert_eq!( + check_command_requirements(xtask_check, false), + (false, false) + ); +} + +#[test] +fn help_is_generated_from_the_same_tables() { + let help = help_text(); + for task in all_tasks() { + assert!(help.contains(&task.name), "missing {}", task.name); + } + assert!(help.contains("Tasks (54):")); + assert!(help.contains("cargo xtask check [RUST_TARGET]")); + assert!(help.contains("cargo xtask update-checksums [--check]")); +} + +#[test] +fn target_identity_distinguishes_native_from_foreign_for_qemu() { + for arch in ARCHES { + assert_eq!( + target_is_foreign(&arch, "x86_64"), + arch.artifact_name != "x86_64" + ); + } + assert!(target_is_foreign( + arch_by_name("x86_64").unwrap(), + "aarch64" + )); +} + +#[test] +fn release_builds_always_use_the_pinned_zig_linker() { + assert_eq!(BUILD_BACKEND, "cargo + Zig 0.16.0"); +} + +#[test] +fn zig_bootstrap_is_version_and_sha256_pinned_for_supported_linux_hosts() { + let expected = [ + ("x86_64", "x86_64-linux"), + ("aarch64", "aarch64-linux"), + ("riscv64", "riscv64-linux"), + ("loongarch64", "loongarch64-linux"), + ]; + for (arch, platform) in expected { + let package = zig_package("linux", arch).unwrap(); + assert_eq!(package.platform, platform); + assert_eq!(package.sha256.len(), 64); + assert!(package.sha256.bytes().all(|byte| byte.is_ascii_hexdigit())); + assert_eq!(package.url, zig_download_url(platform)); + } + assert!(zig_package("windows", "x86_64").is_err()); + assert!(zig_package("linux", "unsupported").is_err()); +} + +#[cfg(unix)] +#[test] +fn relative_program_overrides_are_resolved_against_the_intended_working_directory() { + use std::os::unix::fs::PermissionsExt; + + let root = test_dir("relative-program"); + fs::create_dir_all(&root).unwrap(); + let program = root.join("zig-local"); + fs::write(&program, "#!/bin/sh\nexit 0\n").unwrap(); + fs::set_permissions(&program, fs::Permissions::from_mode(0o755)).unwrap(); + + assert_eq!( + resolve_program(Path::new("./zig-local"), &root).unwrap(), + program + ); + fs::remove_dir_all(root).unwrap(); +} + +#[cfg(unix)] +#[test] +fn rust_sysroot_is_queried_with_the_selected_compiler_from_project_root() { + use std::os::unix::fs::PermissionsExt; + + let root = test_dir("rust-sysroot"); + fs::create_dir_all(&root).unwrap(); + let compiler = root.join("custom-rustc"); + fs::write( + &compiler, + "#!/bin/sh\ntest \"$1 $2\" = '--print sysroot' || exit 2\nprintf '/fixture/sysroot\\n'\n", + ) + .unwrap(); + fs::set_permissions(&compiler, fs::Permissions::from_mode(0o755)).unwrap(); + + assert_eq!( + query_rust_sysroot(&root, OsStr::new("./custom-rustc")).unwrap(), + "/fixture/sysroot" + ); + fs::remove_dir_all(root).unwrap(); +} + +#[cfg(unix)] +#[test] +fn zig_wrapper_maps_all_targets_and_filters_rust_gnu_only_link_args() { + use std::os::unix::fs::PermissionsExt; + + let root = test_dir("zig-wrapper"); + fs::create_dir_all(&root).unwrap(); + let fake_zig = root.join("fake-zig.sh"); + fs::write( + &fake_zig, + "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$FAKE_ZIG_LOG\"\n", + ) + .unwrap(); + fs::set_permissions(&fake_zig, fs::Permissions::from_mode(0o755)).unwrap(); + let wrapper = project_root().join("scripts/zig-linker.sh"); + let rust_sysroot = root.join("rust-sysroot"); + + for arch in ARCHES { + let log = root.join(format!("{}.log", arch.artifact_name)); + let status = Command::new(&wrapper) + .env("URUNTIME_ZIG", &fake_zig) + .env("URUNTIME_RUST_TARGET", arch.rust_target) + .env("URUNTIME_ZIG_TARGET", arch.zig_target) + .env("URUNTIME_RUST_SYSROOT", &rust_sysroot) + .env("FAKE_ZIG_LOG", &log) + .args([ + format!("--target={}", arch.rust_target), + "-Wl,--fix-cortex-a53-843419".into(), + "-nostartfiles".into(), + rust_sysroot + .join("lib/rustlib") + .join(arch.rust_target) + .join("lib/self-contained/crt1.o") + .display() + .to_string(), + "-lc".into(), + "-L".into(), + rust_sysroot + .join("lib/rustlib") + .join(arch.rust_target) + .join("lib") + .display() + .to_string(), + "-Lkeep-me".into(), + "/checkout/self-contained/keep.o".into(), + "-Wl,--wrap=contains--fix-cortex-a53-843419-text".into(), + ]) + .status() + .unwrap(); + assert!(status.success(), "{}", arch.artifact_name); + let actual = fs::read_to_string(log).unwrap(); + assert!(actual.starts_with(&format!("cc\n-target\n{}\n", arch.zig_target))); + assert!(actual.contains("-Lkeep-me\n")); + assert!(!actual.contains( + rust_sysroot + .join("lib/rustlib") + .join(arch.rust_target) + .join("lib") + .to_string_lossy() + .as_ref() + )); + assert!(actual.contains("/checkout/self-contained/keep.o\n")); + assert!(actual.contains("-Wl,--wrap=contains--fix-cortex-a53-843419-text\n")); + assert!(!actual + .lines() + .any(|line| line == "-Wl,--fix-cortex-a53-843419")); + for forbidden in ["unknown-linux", "nostartfiles", "crt1.o", "-lc\n"] { + assert!( + !actual.contains(forbidden), + "{forbidden} leaked for {}: {actual}", + arch.artifact_name + ); + } + } + fs::remove_dir_all(root).unwrap(); +} + +#[cfg(unix)] +#[test] +fn artifact_publish_preserves_cargo_output_and_adds_sections_and_magic_atomically() { + use std::os::unix::fs::PermissionsExt; + + let root = test_dir("publish"); + let sections = root.join("sections"); + fs::create_dir_all(§ions).unwrap(); + fs::write(sections.join("envs"), b"test-envs").unwrap(); + fs::write(sections.join("upd_info"), b"test-update").unwrap(); + let source = env::current_exe().unwrap(); + let destination = root.join("uruntime-appimage-x86_64"); + + publish_artifact(&source, &destination, §ions, *b"AI\x02").unwrap(); + + assert!(source.is_file(), "Cargo output was moved instead of copied"); + assert_ne!( + fs::metadata(&destination).unwrap().permissions().mode() & 0o111, + 0, + "published artifact lost executable bits" + ); + let bytes = fs::read(&destination).unwrap(); + assert_eq!(&bytes[8..11], b"AI\x02"); + let sections_output = Command::new("llvm-readelf") + .arg("--sections") + .arg(&destination) + .output() + .unwrap(); + assert!(sections_output.status.success()); + let table = String::from_utf8(sections_output.stdout).unwrap(); + assert!(table.contains(".envs")); + assert!(table.contains(".upd_info")); + assert_eq!( + fs::read_dir(&root) + .unwrap() + .filter_map(Result::ok) + .filter(|entry| entry.file_name().to_string_lossy().contains(".tmp-")) + .count(), + 0 + ); + fs::remove_dir_all(root).unwrap(); +} + +#[cfg(unix)] +#[test] +fn artifact_publish_replaces_destination_symlink_without_clobbering_its_target() { + use std::os::unix::fs::symlink; + + let root = test_dir("publish-symlink"); + let sections = root.join("sections"); + fs::create_dir_all(§ions).unwrap(); + fs::write(sections.join("envs"), b"test-envs").unwrap(); + let victim = root.join("victim"); + fs::write(&victim, b"do-not-touch").unwrap(); + let destination = root.join("uruntime-appimage-x86_64"); + symlink(&victim, &destination).unwrap(); + + publish_artifact( + &env::current_exe().unwrap(), + &destination, + §ions, + *b"AI\x02", + ) + .unwrap(); + + assert_eq!(fs::read(&victim).unwrap(), b"do-not-touch"); + assert!(!fs::symlink_metadata(&destination) + .unwrap() + .file_type() + .is_symlink()); + fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn checksum_update_uses_validated_payloads_sorted_atomic_check_and_no_change() { + use build_support::{AssetKind, AssetSource, ElfType}; + use xxhash_rust::xxh64::xxh64; + + let target = build_support::target_spec("x86_64-unknown-linux-musl").unwrap(); + let direct = synthetic_helper_elf(3); + let payload = synthetic_helper_elf(2); + let compressed = zstd::stream::encode_all(&payload[..], 3).unwrap(); + let mut wrapper = b"wrapper-prefix".to_vec(); + wrapper.extend_from_slice(&compressed); + wrapper.extend_from_slice(b"SQUEEZE!"); + wrapper.extend_from_slice(&(payload.len() as u64).to_le_bytes()); + wrapper.extend_from_slice(&(compressed.len() as u64).to_le_bytes()); + wrapper.extend_from_slice(&xxh64(&payload, 0).to_le_bytes()); + let sources = vec![ + AssetSource { + target, + name: "z-direct", + url: "fixture://direct".into(), + kind: AssetKind::Direct, + elf_type: ElfType::StaticPie, + }, + AssetSource { + target, + name: "a-wrapper", + url: "fixture://wrapper".into(), + kind: AssetKind::DwarfsWrapper, + elf_type: ElfType::StaticExec, + }, + ]; + let fetch = |source: &AssetSource| match source.url.as_str() { + "fixture://direct" => Ok(direct.clone()), + "fixture://wrapper" => Ok(wrapper.clone()), + _ => Err("unexpected URL".to_string()), + }; + let root = test_dir("checksum-update"); + let manifest = root.join("checksums.txt"); + let zig_index = synthetic_zig_index(); + fs::create_dir_all(&root).unwrap(); + fs::write(&manifest, b"original\n").unwrap(); + + let drift = + update_checksum_manifest_with(&manifest, true, &sources, &fetch, &zig_index).unwrap_err(); + assert!(drift.to_string().contains("drift")); + assert_eq!(fs::read(&manifest).unwrap(), b"original\n"); + + assert_eq!( + update_checksum_manifest_with(&manifest, false, &sources, &fetch, &zig_index).unwrap(), + ManifestUpdate::Updated + ); + let generated = fs::read_to_string(&manifest).unwrap(); + let records = build_support::parse_digest_manifest(&generated).unwrap(); + assert_eq!(records.len(), 2); + assert_eq!(records[0].name, "a-wrapper"); + assert_eq!(records[1].name, "z-direct"); + assert_ne!(records[0].source_sha256, records[0].payload_sha256); + assert_eq!(records[1].source_sha256, records[1].payload_sha256); + assert_eq!( + update_checksum_manifest_with(&manifest, false, &sources, &fetch, &zig_index).unwrap(), + ManifestUpdate::Unchanged + ); + fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn helper_checksum_cache_uses_the_build_source_directory() { + let project = PathBuf::from("/project"); + let target = build_support::target_spec("x86_64-unknown-linux-musl").unwrap(); + let direct = build_support::AssetSource { + target, + name: "squashfuse", + url: "fixture://direct".into(), + kind: build_support::AssetKind::Direct, + elf_type: build_support::ElfType::StaticPie, + }; + let wrapper = build_support::AssetSource { + target, + name: "dwarfs-universal", + url: "fixture://wrapper".into(), + kind: build_support::AssetKind::DwarfsWrapper, + elf_type: build_support::ElfType::StaticExec, + }; + + assert_eq!( + helper_source_cache_path(&project, &direct).unwrap(), + project + .join("assets-x86_64") + .join("squashfuse-0.6.3.r2") + .join("squashfuse") + ); + assert_eq!( + helper_source_cache_path(&project, &wrapper).unwrap(), + project + .join("assets-x86_64") + .join("dwarfs-0.15.7") + .join("dwarfs-universal-wrapper") + ); +} + +#[test] +fn checksum_cache_is_reused_only_when_its_digest_matches() { + let root = test_dir("verified-helper-cache"); + fs::create_dir_all(&root).unwrap(); + let cached = root.join("helper"); + fs::write(&cached, b"release bytes").unwrap(); + let expected = build_support::sha256_hex(b"release bytes"); + + assert_eq!( + read_verified_cached_helper_source(&cached, Some(&expected)).unwrap(), + Some(b"release bytes".to_vec()) + ); + assert_eq!( + read_verified_cached_helper_source(&cached, Some(&"0".repeat(64))).unwrap(), + None + ); + assert_eq!( + read_verified_cached_helper_source(&cached, None).unwrap(), + None + ); + fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn synthetic_zig_index_matches_pinned_manifest_and_resolves_host_archive() { + let mut release = serde_json::Map::new(); + for package in zig_packages().unwrap() { + release.insert( + package.platform, + serde_json::json!({ + "tarball": package.url, + "shasum": package.sha256, + }), + ); + } + let index = serde_json::to_vec(&serde_json::json!({ ZIG_VERSION: release })).unwrap(); + assert!(cached_zig_index_matches_manifest(&index)); + let package = zig_package_from_index(&index, "linux", "x86_64").unwrap(); + assert_eq!(package.platform, "x86_64-linux"); + assert_eq!( + zig_archive_path(Path::new("/cache"), &package), + PathBuf::from(format!( + "/cache/zig-{ZIG_VERSION}-x86_64-linux-{}.tar.xz", + package.sha256 + )) + ); + + let mut changed: serde_json::Value = serde_json::from_slice(&index).unwrap(); + changed[ZIG_VERSION]["x86_64-linux"]["shasum"] = serde_json::Value::String("b".repeat(64)); + assert!(!cached_zig_index_matches_manifest( + &serde_json::to_vec(&changed).unwrap() + )); +} + +#[test] +fn checksum_update_propagates_manifest_read_errors() { + let root = test_dir("checksum-read-error"); + fs::create_dir_all(&root).unwrap(); + let error = update_checksum_manifest_with( + &root, + true, + &[], + &|_| unreachable!(), + &synthetic_zig_index(), + ) + .unwrap_err(); + assert!(error.to_string().contains("failed to read")); + fs::remove_dir_all(root).unwrap(); +} + +fn synthetic_zig_index() -> Vec { + let mut release = serde_json::Map::new(); + for platform in ZIG_PLATFORMS { + release.insert( + platform.to_string(), + serde_json::json!({ + "tarball": zig_download_url(platform), + "shasum": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }), + ); + } + serde_json::to_vec(&serde_json::json!({ ZIG_VERSION: release })).unwrap() +} + +fn synthetic_helper_elf(elf_type: u16) -> Vec { + let mut elf = vec![0u8; 160]; + elf[..4].copy_from_slice(b"\x7fELF"); + elf[4] = 2; + elf[5] = 1; + elf[6] = 1; + elf[16..18].copy_from_slice(&elf_type.to_le_bytes()); + elf[18..20].copy_from_slice(&62u16.to_le_bytes()); + elf[20..24].copy_from_slice(&1u32.to_le_bytes()); + elf[32..40].copy_from_slice(&64u64.to_le_bytes()); + elf[52..54].copy_from_slice(&64u16.to_le_bytes()); + elf[54..56].copy_from_slice(&56u16.to_le_bytes()); + elf[56..58].copy_from_slice(&1u16.to_le_bytes()); + elf[64..68].copy_from_slice(&1u32.to_le_bytes()); + elf[68..72].copy_from_slice(&5u32.to_le_bytes()); + elf[72..80].copy_from_slice(&120u64.to_le_bytes()); + elf[96..104].copy_from_slice(&40u64.to_le_bytes()); + elf[104..112].copy_from_slice(&40u64.to_le_bytes()); + elf +} + +fn test_dir(label: &str) -> PathBuf { + env::temp_dir().join(format!("uruntime-xtask-{label}-{}", std::process::id())) +}