diff --git a/.cargo/config.toml b/.cargo/config.toml index 9f8ff973..9768afc5 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -13,14 +13,6 @@ rustflags = ["-C", "target-feature=+lse", "-C", "target-cpu=native"] [target.aarch64-pc-windows-msvc] rustflags = ["-C", "target-feature=+lse"] -# macOS Apple Silicon -[target.aarch64-apple-darwin] -rustflags = ["-C", "target-feature=+lse", "-C", "target-cpu=native"] - -# macOS Intel -[target.x86_64-apple-darwin] -rustflags = ["-C", "target-cpu=native"] - # Tier B # Linux x86_64 MUSL [target.x86_64-unknown-linux-musl] diff --git a/.changes/avoid-duplicate-ed25519-secret-9748.md b/.changes/avoid-duplicate-ed25519-secret-9748.md new file mode 100644 index 00000000..3cc78711 --- /dev/null +++ b/.changes/avoid-duplicate-ed25519-secret-9748.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Avoid duplicate Ed25519 secret expansion and repeated HKDF prefix-state copies without changing outputs. diff --git a/.changes/bound-password-record-pbkdf2-4c8b.md b/.changes/bound-password-record-pbkdf2-4c8b.md new file mode 100644 index 00000000..0f52bf00 --- /dev/null +++ b/.changes/bound-password-record-pbkdf2-4c8b.md @@ -0,0 +1,8 @@ +--- +"rscrypto" = "patch" +--- + +Bound default password-record PBKDF2 verification work while keeping raw +derivation and compatibility policies unbounded. Custom record policies can +set an explicit ceiling with `verify_with_policy_bounded` or +`verify_password_with_policy_bounded`. diff --git a/.changes/bounded-password-records.md b/.changes/bounded-password-records.md index 13025a31..8aa8d885 100644 --- a/.changes/bounded-password-records.md +++ b/.changes/bounded-password-records.md @@ -1,5 +1,5 @@ --- -"rscrypto" = "minor" +"rscrypto" = "major" --- Argon2 and scrypt password verification now reject noncanonical or over-budget diff --git a/.changes/clear-ecdsa-accelerated-reductio-e464.md b/.changes/clear-ecdsa-accelerated-reductio-e464.md new file mode 100644 index 00000000..990f2447 --- /dev/null +++ b/.changes/clear-ecdsa-accelerated-reductio-e464.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Clear ECDSA accelerated reduction and inversion scratch after use. diff --git a/.changes/clear-ml-kem-secret-c6e5.md b/.changes/clear-ml-kem-secret-c6e5.md new file mode 100644 index 00000000..3b3dbdc6 --- /dev/null +++ b/.changes/clear-ml-kem-secret-c6e5.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Clear ML-KEM secret SHA-3 and SHAKE state after use. diff --git a/.changes/clear-portable-aead-authenticati-fff5.md b/.changes/clear-portable-aead-authenticati-fff5.md new file mode 100644 index 00000000..f04f59b7 --- /dev/null +++ b/.changes/clear-portable-aead-authenticati-fff5.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Clear portable AEAD authentication state after use. diff --git a/.changes/clear-rsa-private-key-f3e1.md b/.changes/clear-rsa-private-key-f3e1.md new file mode 100644 index 00000000..48d2006a --- /dev/null +++ b/.changes/clear-rsa-private-key-f3e1.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Clear RSA private-key validation buffers on every return path. diff --git a/.changes/collapse-rapidhash-to-the-3c61.md b/.changes/collapse-rapidhash-to-the-3c61.md index 59fd4860..4098bcbf 100644 --- a/.changes/collapse-rapidhash-to-the-3c61.md +++ b/.changes/collapse-rapidhash-to-the-3c61.md @@ -1,5 +1,5 @@ --- -"rscrypto" = "minor" +"rscrypto" = "major" --- Collapse RapidHash to portable `RapidHash64`, `RapidStreamHasher`, diff --git a/.changes/correct-aes-gcm-siv-a96a.md b/.changes/correct-aes-gcm-siv-a96a.md new file mode 100644 index 00000000..34062f54 --- /dev/null +++ b/.changes/correct-aes-gcm-siv-a96a.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Correct AES-GCM-SIV RFC input bounds. diff --git a/.changes/correct-cshake-and-kmac-2abd.md b/.changes/correct-cshake-and-kmac-2abd.md new file mode 100644 index 00000000..a3d863e7 --- /dev/null +++ b/.changes/correct-cshake-and-kmac-2abd.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Correct cSHAKE and KMAC byte padding at exact rate boundaries. diff --git a/.changes/correct-x86-and-power-6479.md b/.changes/correct-x86-and-power-6479.md new file mode 100644 index 00000000..50c0a8cd --- /dev/null +++ b/.changes/correct-x86-and-power-6479.md @@ -0,0 +1,6 @@ +--- +"rscrypto" = "patch" +--- + +Correct x86 CRC-16 and CRC-24 four-way tails, x86 and POWER capability detection, and cached initialization after a +panic. diff --git a/.changes/make-cryptographic-boundaries-mi-fba1.md b/.changes/make-cryptographic-boundaries-mi-fba1.md index ad52d513..81242da9 100644 --- a/.changes/make-cryptographic-boundaries-mi-fba1.md +++ b/.changes/make-cryptographic-boundaries-mi-fba1.md @@ -1,5 +1,5 @@ --- -"rscrypto" = "minor" +"rscrypto" = "major" --- Make cryptographic boundaries misuse-resistant: keyed BLAKE2 uses validated borrowed key types and variable outputs fail with typed errors, normal AEAD sealing owns nonce issuance while caller nonces require an expert import, entropy and platform override failures no longer panic, and diagnostic or dangerous capabilities no longer clutter the crate root. diff --git a/.changes/normalize-rsa-crt-exponents-5f58.md b/.changes/normalize-rsa-crt-exponents-5f58.md new file mode 100644 index 00000000..16dfeb0f --- /dev/null +++ b/.changes/normalize-rsa-crt-exponents-5f58.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Normalize RSA CRT exponents before private operations. diff --git a/.changes/owned-secret-equality.md b/.changes/owned-secret-equality.md index b3fd3b9b..d0c27b16 100644 --- a/.changes/owned-secret-equality.md +++ b/.changes/owned-secret-equality.md @@ -1,5 +1,5 @@ --- -"rscrypto" = "minor" +"rscrypto" = "major" --- Secret comparison is now owned by fixed-size cryptographic key, tag, and diff --git a/.changes/preserve-a-scalar-declassificati-452a.md b/.changes/preserve-a-scalar-declassificati-452a.md new file mode 100644 index 00000000..6789552d --- /dev/null +++ b/.changes/preserve-a-scalar-declassificati-452a.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Preserve a scalar declassification boundary for fixed-size secret-owner equality so optimized x86 builds remain covered by constant-time binary proofs. diff --git a/.changes/rebind-reusable-rsa-scratch-5003.md b/.changes/rebind-reusable-rsa-scratch-5003.md new file mode 100644 index 00000000..6cea1956 --- /dev/null +++ b/.changes/rebind-reusable-rsa-scratch-5003.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Rebind reusable RSA scratch state when switching between same-width keys. diff --git a/.changes/reject-sha-384-sha-62b4.md b/.changes/reject-sha-384-sha-62b4.md new file mode 100644 index 00000000..b5432f7b --- /dev/null +++ b/.changes/reject-sha-384-sha-62b4.md @@ -0,0 +1,6 @@ +--- +"rscrypto" = "patch" +--- + +Panic before absorbing SHA-384, SHA-512, or SHA-512/256 input that would +exceed the FIPS 180-4 length field instead of wrapping the encoded bit length. diff --git a/.changes/reject-weak-rsa-private-factors-a8c2.md b/.changes/reject-weak-rsa-private-factors-a8c2.md new file mode 100644 index 00000000..6ea093f2 --- /dev/null +++ b/.changes/reject-weak-rsa-private-factors-a8c2.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "major" +--- + +Reject RSA private-key imports with nonstandard-width factors or factors that fail probable-prime screening. diff --git a/.changes/require-security-strength-kmac-8015.md b/.changes/require-security-strength-kmac-8015.md new file mode 100644 index 00000000..465fc9d3 --- /dev/null +++ b/.changes/require-security-strength-kmac-8015.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Require security-strength KMAC tags in default verification APIs and provide explicit primitive verification for protocol-defined shorter tags. diff --git a/.changes/seal-secret-bearing-comparison-4815.md b/.changes/seal-secret-bearing-comparison-4815.md index 14fd1aa3..887e4c45 100644 --- a/.changes/seal-secret-bearing-comparison-4815.md +++ b/.changes/seal-secret-bearing-comparison-4815.md @@ -1,5 +1,5 @@ --- -"rscrypto" = "minor" +"rscrypto" = "major" --- Secret-bearing fixed-size keys, shared secrets, keypairs, authentication tags, diff --git a/.changes/secret-bearing-hmac-hkdf-8528.md b/.changes/secret-bearing-hmac-hkdf-8528.md index d0f66149..87eefa94 100644 --- a/.changes/secret-bearing-hmac-hkdf-8528.md +++ b/.changes/secret-bearing-hmac-hkdf-8528.md @@ -1,5 +1,5 @@ --- -"rscrypto" = "minor" +"rscrypto" = "major" --- Secret-bearing HMAC, HKDF, KMAC, and PBKDF2 states no longer implement diff --git a/.changes/typed-jwt-verifier.md b/.changes/typed-jwt-verifier.md index 8a8d81af..7c2d42c7 100644 --- a/.changes/typed-jwt-verifier.md +++ b/.changes/typed-jwt-verifier.md @@ -1,5 +1,5 @@ --- -"rscrypto" = "minor" +"rscrypto" = "major" --- RSA JWT/JWS verification is now bound to one verifier-owned diff --git a/.changes/verify-imported-assembly-and-7c0b.md b/.changes/verify-imported-assembly-and-7c0b.md new file mode 100644 index 00000000..63d8a5a3 --- /dev/null +++ b/.changes/verify-imported-assembly-and-7c0b.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Verify imported assembly and cryptographic test vectors against immutable upstream sources and deterministic transforms. diff --git a/.github/workflows/_ci-suite.yaml b/.github/workflows/_ci-suite.yaml index 65997b27..4f32be50 100644 --- a/.github/workflows/_ci-suite.yaml +++ b/.github/workflows/_ci-suite.yaml @@ -84,7 +84,7 @@ jobs: uses: ./.github/workflows/_rust-job.yaml with: runner: runs-on=${{ github.run_id }}/runner=linux-x64-ci - timeout_minutes: 90 + timeout_minutes: 120 cache_key: ${{ inputs.cache_key_prefix }}-feature-contracts tools_mode: minimal toolchain_components: "rust-src" @@ -140,6 +140,19 @@ jobs: operation: native-ibm test_mode: ${{ inputs.test_mode }} + platform-amx: + name: Platform (Linux x64 AMX permission) + uses: ./.github/workflows/_rust-job.yaml + with: + runner: runs-on=${{ github.run_id }}/runner=intel-spr + timeout_minutes: 30 + cache_key: ${{ inputs.cache_key_prefix }}-platform-amx + tools_mode: none + toolchain_components: "rustfmt" + enable_magic_cache: true + enable_rust_cache: true + operation: platform-amx + cross-targets: name: Cross Targets uses: ./.github/workflows/_rust-job.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4111c30d..1c37f6b4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -47,18 +47,28 @@ jobs: enable-magic-cache: false enable-rust-cache: true + - name: Build Plan (PR) + if: github.event_name == 'pull_request' + id: rail + uses: loadingalias/cargo-rail-action@f622a3936a231fe78a772292c6892d71e8c57f9f # v6.1.0 + with: + version: "0.20.0" + checksum: required + since: ${{ github.event.pull_request.base.sha }} + - name: Resolve Build Plan if: always() id: resolve env: - RAIL_PLAN_STEP_OUTCOME: ${{ steps.rail_setup.outcome }} - RAIL_SINCE: origin/${{ github.base_ref }} + RAIL_PLAN_STEP_OUTCOME: ${{ steps.rail.outcome }} + RAIL_SCOPE_JSON: ${{ steps.rail.outputs.scope-json }} + RAIL_SURFACES_JSON: ${{ steps.rail.outputs.surfaces-json }} run: scripts/ci/resolve-rail-plan.sh - name: Check Release Intent Coverage if: github.event_name == 'pull_request' env: - RAIL_BASE_REF: origin/${{ github.base_ref }} + RAIL_BASE_REF: ${{ steps.rail.outputs.base-ref }} run: cargo rail change check --since "$RAIL_BASE_REF" --required dependency-smoke: diff --git a/.github/workflows/rsa.yaml b/.github/workflows/rsa.yaml index 393e1b0c..3204ae2f 100644 --- a/.github/workflows/rsa.yaml +++ b/.github/workflows/rsa.yaml @@ -58,9 +58,25 @@ jobs: artifact_path: ci-evidence/rsa-leakage-${{ matrix.target.name }}.log artifact_always: true + rsa-linux-x86_64-asm: + name: RSA Assembly (Linux x86-64) + uses: ./.github/workflows/_rust-job.yaml + with: + runner: runs-on=${{ github.run_id }}/runner=linux-x64-ci + timeout_minutes: 45 + cache_key: rsa-linux-x86_64-asm + tools_mode: none + toolchain_components: "" + enable_magic_cache: true + enable_rust_cache: true + operation: rsa-linux-x64-asm + artifact_name: rsa-linux-x86_64-asm + artifact_path: ci-evidence/rsa-linux-x86_64-asm.log + artifact_always: true + complete: name: Complete (RSA) - needs: [rsa-miri, rsa-leakage] + needs: [rsa-miri, rsa-leakage, rsa-linux-x86_64-asm] if: always() runs-on: ubuntu-latest steps: @@ -69,7 +85,8 @@ jobs: failed=0 for lane in \ "rsa-miri=${{ needs.rsa-miri.result }}" \ - "rsa-leakage=${{ needs.rsa-leakage.result }}" + "rsa-leakage=${{ needs.rsa-leakage.result }}" \ + "rsa-linux-x86_64-asm=${{ needs.rsa-linux-x86_64-asm.result }}" do name="${lane%%=*}" result="${lane#*=}" diff --git a/.github/workflows/weekly.yaml b/.github/workflows/weekly.yaml index 7eccc921..79d9e619 100644 --- a/.github/workflows/weekly.yaml +++ b/.github/workflows/weekly.yaml @@ -91,7 +91,7 @@ jobs: fuzz-output/corpus.tar.gz fuzz/artifacts/ fuzz-packages/*/artifacts/ - artifact_always: false + artifact_always: true asan-fuzzing: name: ASan Fuzz Corpus (${{ matrix.target.name }}) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ad234c7a..656ebf9a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,7 @@ Use a short-lived feature branch for every change. `main` is the protected, releasable history; it is not a working branch. -## What the gates do +## Gate model | Action | Purpose | |---|---| @@ -17,20 +17,25 @@ actors, so merged commits do not repeat the pull-request suite. Release candidates get a separate exact-commit Weekly run with the complete CI suite, Cargo graph assurance, and release evidence. -## Daily branch workflow +## Create a branch -Start from current `main`: +Start from a clean worktree and current `main`: ```bash +git status --short git switch main git pull --ff-only git switch -c ``` -Make one focused change. Add a `.changes/*.md` file when crate users will -observe the result: an API, behavior, security, performance, compatibility, or -release-artifact change. Internal-only tooling and maintainer-documentation -changes normally do not need one. +Do not discard unrelated local changes to run this sequence. Preserve them or +move them to their own branch first. + +## Record release intent + +Add a `.changes/*.md` file when crate users will observe an API, behavior, +security, performance, compatibility, or release-artifact change. Internal +tooling and maintainer-only documentation normally do not need one. ```bash just release-change patch "Describe the user-visible result." @@ -40,6 +45,8 @@ Use `minor` or `major` instead of `patch` when the compatibility impact requires it. The pre-push check is the final authority on whether release intent is missing. +## Validate the change + Run checks proportional to the change. Common starting points are: ```bash @@ -62,9 +69,12 @@ Use deeper checks where the risk requires them: | `unsafe`, SIMD, ASM, or dispatch | Backend equivalence tests and `just test-fuzz-asan --all` where the target runs natively | | Portable unsafe path | `just test-miri` | | Constant-time claim boundary | `just ct-full --target `; update `ct.toml` only with matching evidence | +| Apple Silicon RSA assembly | `just test-rsa-macos-asm` on a physical local Arm64 Mac; GitHub Actions intentionally has no macOS runner | | Public API change | `cargo semver-checks --package rscrypto --all-features` | | Dependency or release change | `cargo deny check all` and `cargo audit --ignore RUSTSEC-2023-0071` | +## Review and commit + Inspect and commit only the intended files: ```bash @@ -75,6 +85,8 @@ git diff --cached git commit -m "module: imperative outcome" ``` +## Push and open a pull request + Push the current branch with its upstream. No extra Git flags are needed: ```bash @@ -85,18 +97,20 @@ just push current branch. No rscrypto Git-hook installation is required. Use `just push-full` when the change is unusually broad or release-sensitive. -Open a draft pull request while work is still changing; expensive jobs wait -until the pull request is ready for review. A branch push alone does not start -the normal PR suite: +Open a draft pull request while the change is still in progress. Expensive jobs +wait until the pull request is ready for review; a branch push alone does not +start the normal pull-request suite: ```bash gh pr create --base main --fill --draft ``` Mark the pull request ready only when its head is ready for CI, then wait for -the required `Complete` check. Because the repository currently has one -maintainer, no second approval is required; review the final diff yourself, -resolve any open threads, and merge in the GitHub UI. +the required `Complete` check. Resolve every open review thread and review the +final diff before merging in the GitHub UI. GitHub enforces the current +approval policy. + +## Clean up after merge After GitHub reports the pull request merged: diff --git a/Cargo.lock b/Cargo.lock index 8e1e853c..06d48440 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,9 +24,9 @@ dependencies = [ [[package]] name = "aegis" -version = "0.9.12" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e07d39d15384924b35b70d7b8fa1f9a2934101dd3fa4722ede163cc4f9b7b960" +checksum = "58541132f980da31e9aa99f7bdee69bc84bf1e168b9b91ef2dbe8abb7b4ce5dd" dependencies = [ "cc", "softaes", @@ -45,9 +45,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138" +checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" dependencies = [ "cipher 0.5.2", "cpubits", @@ -61,7 +61,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" dependencies = [ "aead 0.6.1", - "aes 0.9.1", + "aes 0.9.2", "cipher 0.5.2", "ctr 0.10.1", "ghash", @@ -173,11 +173,11 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.1" +version = "1.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" dependencies = [ - "aws-lc-sys 0.42.0", + "aws-lc-sys 0.43.0", "untrusted 0.7.1", "zeroize", ] @@ -196,9 +196,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.42.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" dependencies = [ "cc", "cmake", @@ -242,7 +242,7 @@ dependencies = [ "regex", "rustc-hash", "shlex 1.3.0", - "syn", + "syn 2.0.119", ] [[package]] @@ -262,9 +262,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "blake2" @@ -329,9 +329,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -430,9 +430,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.8.1" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" dependencies = [ "glob", "libc", @@ -441,18 +441,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" dependencies = [ "anstyle", "clap_lex", @@ -565,7 +565,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ "digest 0.10.7", - "spin 0.10.0", + "spin 0.10.1", ] [[package]] @@ -707,6 +707,17 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "cshake" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6250a2d96a09edbe8e75ed29c87d05512ee2cbb24c7e8c684657f7930ffd3c6" +dependencies = [ + "digest 0.11.3", + "keccak 0.2.0", + "sponge-cursor", +] + [[package]] name = "ctr" version = "0.9.2" @@ -759,7 +770,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -887,9 +898,9 @@ dependencies = [ [[package]] name = "either" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] name = "elliptic-curve" @@ -942,9 +953,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "ff" @@ -1004,21 +1015,21 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-core", "futures-task", @@ -1078,14 +1089,14 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ - "polyval 0.7.2", + "polyval 0.7.3", ] [[package]] name = "glob" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "group" @@ -1141,7 +1152,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1186,9 +1197,9 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "ctutils", "subtle", @@ -1285,14 +1296,14 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.9.8", + "spin 0.9.9", ] [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libcrux-intrinsics" @@ -1671,9 +1682,9 @@ dependencies = [ [[package]] name = "polyval" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b20f20e954175de5f463f67781b35583397d916b1d148738923711b2ad16bee8" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" dependencies = [ "cpubits", "cpufeatures 0.3.0", @@ -1696,7 +1707,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.119", ] [[package]] @@ -1754,14 +1765,14 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -1793,9 +1804,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -1927,9 +1938,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -1939,9 +1950,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -2030,6 +2041,7 @@ dependencies = [ "crc32fast", "crc64fast", "criterion", + "cshake", "digest 0.11.3", "dryoc", "ed25519-dalek", @@ -2172,9 +2184,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -2182,29 +2194,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -2328,15 +2340,15 @@ checksum = "45e14297decde697ddf377c25752aead0927d5cfc89c2684d2af96901a4ceeea" [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" [[package]] name = "spin" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" [[package]] name = "spki" @@ -2372,9 +2384,20 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -2465,9 +2488,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "uuid" -version = "1.23.5" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -2546,7 +2569,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.119", "wasm-bindgen-shared", ] @@ -2719,28 +2742,28 @@ dependencies = [ [[package]] name = "xxhash-rust" -version = "0.8.16" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "zerocopy" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2760,7 +2783,7 @@ checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index af80c93d..2ae1479b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ indexing_slicing = "allow" undocumented_unsafe_blocks = "deny" missing_safety_doc = "deny" +chunks_exact_to_as_chunks = "allow" [workspace.lints.rustdoc] broken_intra_doc_links = "deny" @@ -156,6 +157,7 @@ criterion = { version = "^0.8.2", default-features = false, features = [ proptest = "^1.11.0" # Oracles +cshake = { version = "0.2.1", default-features = false } crc = "3.4.0" crc-fast = { version = "1.10.0", default-features = false, features = ["std"] } crc32c = "0.6.8" @@ -171,9 +173,9 @@ sha2 = "0.11.0" sha2_010 = { package = "sha2", version = "0.10.9" } sha3 = "0.12.0" keccak = "0.2.0" -xxhash-rust = { version = "0.8.16", features = ["xxh3"] } +xxhash-rust = { version = "0.8.18", features = ["xxh3"] } rapidhash = "=4.5.1" -tiny-keccak = { version = "2.0.2", features = ["cshake", "kmac", "shake"] } +tiny-keccak = { version = "2.0.2", features = ["kmac", "shake"] } digest = { version = "0.11.3", features = ["dev"] } p256 = { version = "0.14.0", default-features = false, features = ["ecdsa", "pkcs8", "std"] } p384 = { version = "0.13.1", default-features = false, features = ["ecdsa", "pkcs8", "std"] } @@ -186,7 +188,7 @@ hkdf = "0.13.0" chacha20poly1305 = "0.11.0" aes-gcm = "0.11.0" aes-gcm-siv = "0.11.1" -aegis = "0.9.12" +aegis = "0.9.15" ascon-aead = { version = "0.6", default-features = false, features = ["alloc"] } serde_json = "1" ring = "0.17.14" @@ -217,7 +219,7 @@ libcrux-ml-kem = { version = "0.0.10", default-features = false, features = [ ] } [target.'cfg(all(any(unix, windows), not(target_arch = "wasm32"), not(any(target_arch = "s390x", target_arch = "powerpc64"))))'.dev-dependencies] -aws-lc-rs = "1.17.1" +aws-lc-rs = "1.17.3" aws-lc-sys = "0.41.0" [lib] diff --git a/README.md b/README.md index 47626d06..afdb91d1 100644 --- a/README.md +++ b/README.md @@ -10,29 +10,12 @@ `rscrypto` provides pure Rust cryptographic primitives, cryptographic and fast hashes, password hashing, and checksums behind one feature model. -It has no production C/FFI, OpenSSL, or system-library dependency. Leaf features -support narrow builds; umbrella features compose larger surfaces. +It has no production C/FFI, OpenSSL, or system-library dependency. Enable one +leaf feature for one primitive, an umbrella feature for a family, or `full` for +the complete primitive surface. Every supported target retains the portable +Rust backend; SIMD and assembly only accelerate it. -Use one leaf feature for one primitive, a group for a subset of primitives, or `full` for the full crate surface. The portable Rust backend is always present. SIMD and ASM are only accelerators. - -**Published benchmark snapshot:** `1.59x` geomean across the Linux runners vs the fastest-external competitors with `4,052 / 6,750` wins and `6,101 / 6,750` wins-or-ties. - -macOS Apple Silicon local evidence: `1.37x` geomean vs fastest-external competitors with `382 / 774` wins and `708 / 774` wins-or-ties. - -Raw runs, methodology, and known losses are in -[`benchmark_results/OVERVIEW.md`](benchmark_results/OVERVIEW.md). - -

- rscrypto benchmark chart: 1.59x Linux and 1.37x Apple Silicon fastest-matched geomeans, checksums at 5.18x against crc-fast, crc, crc32fast, crc32c, and crc64fast, plus primitive geomean bars and M1 MBP Apple Silicon notes. -

- -

- Chart: benchmark scorecard. Values above 1.00x mean rscrypto is faster than the fastest matched external implementation. -

- -## Why rscrypto? +## Scope - One feature model for hashes, MACs, KDFs, password hashing, AEADs, signatures, key exchange, ML-KEM, RSA, and checksums. @@ -50,7 +33,7 @@ Raw runs, methodology, and known losses are in store, or protocol implementation. It does not claim FIPS 140-3 validation, a third-party audit, formal verification, or whole-crate constant-time behavior. -## Install +## Choose features Minimal `no_std` SHA-2 build: @@ -66,9 +49,12 @@ Full primitive stack with OS randomness enabled: rscrypto = { version = "0.7.8", features = ["full", "getrandom"] } ``` -Use `default-features = false` for `no_std` builds. Enable `getrandom` only when you need APIs that generate salts, keys, nonces, or RSA key-gen entropy from the operating system. +Use `default-features = false` for `no_std` builds. Enable `getrandom` only for +APIs that obtain salts, keys, nonces, or RSA key-generation entropy from the +operating system. See [`docs/features.md`](docs/features.md) for exact feature +dependencies and deployment controls. -## Quick Start +## Quick start ```rust use rscrypto::Sha256; @@ -84,48 +70,50 @@ assert_eq!(h.finalize(), one_shot); The common API shape is one-shot when convenient and streaming when needed. -## Common Workflows +## Common workflows -| Task | Feature | Start Here | +| Task | Features | Start here | |---|---|---| -| AEAD seal/open | `chacha20poly1305,getrandom` | [`examples/aead_seal_open.rs`](examples/aead_seal_open.rs) | -| Ed25519 and ECDSA signatures | `ed25519,ecdsa-p256,getrandom` | [`examples/signatures.rs`](examples/signatures.rs) | +| AEAD seal/open | `chacha20poly1305`, `getrandom` | [`examples/aead_seal_open.rs`](examples/aead_seal_open.rs) | +| Ed25519 and ECDSA signatures | `ed25519`, `ecdsa-p256`, `getrandom` | [`examples/signatures.rs`](examples/signatures.rs) | | RSA-PSS verification | `rsa` | [`examples/rsa_pss_verify.rs`](examples/rsa_pss_verify.rs) | -| ML-KEM shared secret | `ml-kem,getrandom` | [`examples/mlkem_encapsulation.rs`](examples/mlkem_encapsulation.rs) | -| Argon2id and scrypt password hashing | `password-hashing,getrandom` | [`examples/password_hashing.rs`](examples/password_hashing.rs) | +| ML-KEM shared secret | `ml-kem`, `getrandom` | [`examples/mlkem_encapsulation.rs`](examples/mlkem_encapsulation.rs) | +| Argon2id and scrypt password hashing | `password-hashing`, `getrandom` | [`examples/password_hashing.rs`](examples/password_hashing.rs) | Use [`docs/types.md`](docs/types.md) when you need the full type map, and [`docs/features.md`](docs/features.md) when you need the smallest feature set. -## What You Get +## Capabilities -| Need | Included | Feature Path | +| Need | Included | Feature path | |---|---|---| -| Cryptographic Hashes | SHA-2, SHA-3, SHAKE, cSHAKE128/256, BLAKE2, BLAKE3, Ascon-Hash/XOF/CXOF | `hashes` or leaf features | -| MACs & KDFs | HMAC-SHA-2/SHA-3, KMAC128/256, standalone Poly1305, HKDF-SHA-2, PBKDF2-HMAC-SHA-2 | `auth` or leaf features | -| Password Hashing | Raw Argon2d/i/id and scrypt KDFs; generated, bounded PHC password records | `auth`, `argon2`, `scrypt`, `phc-strings` | -| Public-Key Primitives | ECDSA P-256/P-384 signing/verification, Ed25519 signatures, RSA signing/verification/OAEP/RSAES-PKCS1-v1_5/key generation, X25519 key exchange, ML-KEM-512/768/1024 KEMs | `auth`, `signatures`, `key-exchange`, `ecdsa`, `ecdsa-p256`, `ecdsa-p384`, `ed25519`, `rsa`, `x25519`, `ml-kem` | -| AEAD Encryption | AES-128/256-GCM, AES-128/256-GCM-SIV, ChaCha20-Poly1305, XChaCha20-Poly1305, AEGIS-256, Ascon-AEAD128 | `aead` or leaf features | +| Cryptographic hashes | SHA-2, SHA-3, SHAKE, cSHAKE128/256, BLAKE2, BLAKE3, Ascon-Hash/XOF/CXOF | `hashes` or leaf features | +| MACs and KDFs | HMAC-SHA-2/SHA-3, KMAC128/256, standalone Poly1305, HKDF-SHA-2, PBKDF2-HMAC-SHA-2 | `macs`, `kdfs`, or leaf features | +| Password hashing | Raw Argon2d/i/id and scrypt KDFs; generated, bounded PHC password records | `password-hashing` or leaf features | +| Public-key primitives | ECDSA P-256/P-384 signing/verification, Ed25519 signatures, RSA signing/verification/OAEP/RSAES-PKCS1-v1_5/key generation, X25519 key exchange, ML-KEM-512/768/1024 KEMs | `signatures`, `key-exchange`, or leaf features | +| AEAD encryption | AES-128/256-GCM, AES-128/256-GCM-SIV, ChaCha20-Poly1305, XChaCha20-Poly1305, AEGIS-256, Ascon-AEAD128 | `aead` or leaf features | | Checksums | CRC-16, CRC-24, CRC-32, CRC-32C, CRC-64/XZ, CRC-64/NVMe | `checksums` or leaf features | -| Fast Hashes | XXH3-64/128, RapidHash V3-64 | `xxh3`, `rapidhash` | +| Fast hashes | XXH3-64/128, RapidHash V3-64 | `xxh3`, `rapidhash` | -Flags are layered by use: +Feature layers: -- **Leaf Primitives:** `sha2`, `blake3`, `aes-gcm`, `ed25519`, `x25519`, `ml-kem`, `crc32`, etc. -- **Families/Groups:** `hashes`, `checksums`, `macs`, `kdfs`, `password-hashing`, `aead`, `signatures`, `key-exchange`. -- **Deployment Controls:** `std`, `alloc`, `getrandom`, `parallel`, `serde`, `portable-only`; `serde-secrets` explicitly opts secret material into `serde`. +- Leaf primitives: `sha2`, `blake3`, `aes-gcm`, `ed25519`, `x25519`, + `ml-kem`, `crc32`, and the other algorithm features. +- Families: `hashes`, `checksums`, `macs`, `kdfs`, `password-hashing`, + `aead`, `signatures`, and `key-exchange`. +- Deployment controls: `std`, `alloc`, `getrandom`, `parallel`, `serde`, and + `portable-only`. `serde-secrets` explicitly opts secret material into Serde. -Full Feature Inventory: [`docs/features.md`](docs/features.md). -Public Type Inventory: [`docs/types.md`](docs/types.md). +See the complete [`feature inventory`](docs/features.md) and +[`public type inventory`](docs/types.md). -## Constant-Time Boundaries +## Security -`rscrypto` makes only release-bound, scoped constant-time claims for -secret-bearing operations, not for every function in the crate. `ct.toml` -records the candidate primitive/configuration set; it does not create a public -claim by itself. A claim exists only where the matching signed GitHub release -includes an attested `rscrypto-X.Y.Z-ct-evidence.tar.gz` bundle that passes all -required gates for that exact version, commit, target, profile, and feature set. +Constant-time claims are release-bound and configuration-specific. A claim +exists only when the matching signed GitHub release includes an attested +`rscrypto-X.Y.Z-ct-evidence.tar.gz` bundle whose required gates pass for the +exact version, commit, target, profile, and feature set. [`ct.toml`](ct.toml) +records candidate surfaces; it does not establish a claim by itself. Secret-bearing fixed-size owners do not implement `PartialEq` or `Eq`. Their `ct_eq` methods return an opaque `CtDecision`; callers must explicitly consume @@ -133,39 +121,11 @@ it with `declassify()` to obtain a branchable bit. Verification APIs keep that boundary internal and return one opaque `Result`. This is misuse resistance at the Rust API boundary, not proof about downstream machine code. -The main candidate secret-bearing surfaces in [`ct.toml`](ct.toml) are -MAC/tag verification, AEAD authentication failure shape, X25519 scalar -multiplication, Ed25519 signing and secret public-key derivation, ECDSA -P-256/P-384 blinded signing, ML-KEM-512/768/1024 key gen, -encapsulation, decapsulation secret surfaces, RSA private sign/decrypt leaves, -and selected password-verification comparisons. - Public parsing, unlisted key gen, OS randomness, raw hashes, checksums, non-cryptographic hashes, benchmark paths, and public-key verification math are -not blanket constant-time claims. See [`docs/constant-time.md`](docs/constant-time.md) -for the exact claim and verification model and [`docs/compliance.md`](docs/compliance.md) -for review boundaries. Releases through `v0.6.4` do not contain this bundle and -therefore carry no release-bound constant-time claim. - -## Portability & Accel - -`rscrypto` keeps the portable Rust path as the byte-for-byte authority. ISA kernels are selected only when the target and runtime CPU support them. +outside that claim. Releases through `v0.6.4` contain no CT evidence bundle and +carry no release-bound constant-time claim. -| Target family | Acceleration examples | -|---|---| -| x86 / x86_64 | SSE4.2, AVX2, AVX-512, AES-NI, SHA-NI, VAES, VPCLMULQDQ | -| Arm / AArch64 / Apple Silicon | NEON, AES, PMULL, SHA2, SHA3, SVE2-PMULL | -| IBM Z | CPACF, MSA, VGFM, z/Vector ML-KEM arithmetic | -| POWER / ppc64le | POWER8/9/10 vector and crypto extensions | -| RISC-V | RVV, Zbc, Zvkned, Zvbc | -| WASM | SIMD128 where available, portable fallback everywhere | - -Full platform matrix: [`docs/platforms.md`](docs/platforms.md). - -## Security - -`rscrypto` makes scoped constant-time claims only when a matching release -publishes the required evidence bundle, never for every API or build. The fixed-size secret owners named in [`docs/secret-ownership.md`](docs/secret-ownership.md) overwrite their owned bytes on drop and mask `Debug`; the claim does not extend to caller copies. @@ -175,10 +135,48 @@ crates.io Trusted Publishing, and covered by GitHub build provenance attestations. No third-party audit, FIPS 140-3 certificate, or formal whole-crate proof is -claimed. Report vulnerabilities through +claimed. Read the exact [constant-time model](docs/constant-time.md), +[threat model](THREAT_MODEL.md), and [compliance boundary](docs/compliance.md) +before making a security or assurance claim. Report vulnerabilities through [GitHub Private Vulnerability Reporting](https://github.com/loadingalias/rscrypto/security/advisories/new) or [`SECURITY.md`](SECURITY.md), not public issues. +## Platforms + +The portable Rust implementation is the byte-for-byte authority. Compile-time +target support and, with `std`, detected runtime CPU capabilities select +eligible SIMD or assembly kernels. Unsupported acceleration falls back to +portable Rust. + +See [`docs/platforms.md`](docs/platforms.md) for the dispatch model, target +matrix, and `no_std` coverage. See +[`docs/features.md#portable-only`](docs/features.md#portable-only) before using +`portable-only`; it constrains runtime dispatch but does not remove accelerated +code from a binary. + +## Performance + +The published 2026-07-04 benchmark snapshot is historical. Its aggregate +geomeans are not equivalent-work performance claims because the historical +RustCrypto HMAC-SHA-256 rows included key setup while the compared rscrypto, +`ring`, and AWS-LC rows reused keyed state. The benchmark source now aligns +that setup, but a new aggregate requires a complete regenerated artifact. + +
+Historical 2026-07-04 scorecard (not an equivalent-work aggregate) +

+ rscrypto benchmark chart: 1.59x Linux and 1.37x Apple Silicon fastest-matched geomeans, checksums at 5.18x against crc-fast, crc, crc32fast, crc32c, and crc64fast, plus primitive geomean bars and M1 MBP Apple Silicon notes. +

+
+ +Use individual shape-compatible rows for investigation and benchmark the +deployment workload on its target hardware. The correction, raw results, +methodology, and known losses are in +[`benchmark_results/OVERVIEW.md`](benchmark_results/OVERVIEW.md) and +[`docs/benchmarking.md`](docs/benchmarking.md). + ## Docs - Start: [docs.rs](https://docs.rs/rscrypto), [`examples/`](examples/), @@ -195,9 +193,11 @@ or [`SECURITY.md`](SECURITY.md), not public issues. ## MSRV -Rust **1.91.0**. +The minimum supported Rust version is **1.91.0**. -The pinned nightly in [`rust-toolchain.toml`](rust-toolchain.toml) is used for Miri, fuzzing, and exotic-architecture checks. +The pinned development nightly in +[`rust-toolchain.toml`](rust-toolchain.toml) is separate from the MSRV and is +used for Miri, fuzzing, and architecture-specific checks. ## License diff --git a/SECURITY.md b/SECURITY.md index 1cfd4768..79a45fa7 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,11 +1,10 @@ # Security Policy -## Reporting A Vulnerability +## Report a vulnerability -Please do not report real-world vulnerabilities through public GitHub issues. - -Use GitHub's +Report real-world vulnerabilities through GitHub [Private Vulnerability Reporting](https://github.com/loadingalias/rscrypto/security/advisories/new). +Do not open a public issue. Include: @@ -16,6 +15,9 @@ Include: - Whether exploitability depends on features such as `std`, `alloc`, `getrandom`, `serde`, `serde-secrets`, or hardware acceleration. +Do not include live keys, credentials, personal data, or other secrets. Replace +them with a minimal synthetic reproducer. + You should receive an acknowledgment within 72 hours. If you do not, follow up on the private advisory thread. @@ -45,33 +47,34 @@ Out of scope: - Bugs caused by downstream crates using `rscrypto` outside its documented contract. -## Supported Versions +## Supported versions | Version | Supported | | ------- | --------- | | `0.7.x` | Yes | -Only the latest published minor release receives security patches. Users should -stay current. +Only the latest patch release in the current minor line receives security +fixes. Upgrade to the latest published `0.7.x` release before reporting an issue +that may already be fixed. -## Security Posture +## Security posture `rscrypto` is a pure Rust primitive crate with no mandatory production C/FFI dependency. External crypto crates used for testing, fuzzing, migration checks, or benchmarks are not production dependencies. -Constant-time claims are scoped. They apply only to named secret-bearing -operations and target configurations, not to every API or every build. The -README carries the public security summary; [`docs/constant-time.md`](docs/constant-time.md) -is the exact claim model. The security boundary, adversary model, and review -priorities are documented in [`THREAT_MODEL.md`](THREAT_MODEL.md). +Constant-time claims apply only to named secret-bearing operations and exact +release configurations. They do not cover every API or build. Read +[`docs/constant-time.md`](docs/constant-time.md) for the claim model and +[`THREAT_MODEL.md`](THREAT_MODEL.md) for the security boundary, adversaries, +and review priorities. No third-party security audit, FIPS 140-3 validation, or formal proof is claimed. -## Advisory Packet +## Advisory contents -For a valid vulnerability, the advisory should include: +When a report is valid, the project advisory records: - Affected `rscrypto` versions and feature flags. - Impacted primitives or parsing surfaces. @@ -80,14 +83,14 @@ For a valid vulnerability, the advisory should include: - Severity, patched version, and credit preference. - CVSS and CWE when they are clear enough to be useful. -## AI-Assisted Reports +## AI-assisted reports AI-assisted reports are welcome when they are reproducible. If AI or automated analysis helped produce the report, disclose the tool or model used and include the concrete inputs, outputs, traces, or reproduction steps that support the finding. -## Response Process +## Response process 1. Triage within 72 hours. 2. Reproduce the issue and assess severity. @@ -97,14 +100,15 @@ finding. The default disclosure window is 30 days from the initial report unless a different timeline is agreed on in the private advisory. -## Safe Harbor +## Safe harbor Good-faith research is welcome when it avoids privacy violations, data destruction, service interruption, and access to third-party systems. Do not exploit a vulnerability beyond what is necessary to demonstrate impact. -Researchers who follow this policy will not face legal action from this project -for the reported activity. +The project does not intend to pursue legal action for research conducted and +reported in accordance with this policy. This statement cannot bind third +parties. ## Acknowledgments diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index 5ee3a874..6736bf2f 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -1,7 +1,8 @@ # Threat Model -This is the security-review entry point for `rscrypto`. It defines what the -crate defends, what the caller owns, and where an audit should focus. +Use this document to scope a security review of `rscrypto`. It defines the +crate boundary, protected assets, caller responsibilities, threat assumptions, +evidence, and review priorities. Related documents: [`SECURITY.md`](SECURITY.md) for vulnerability reporting, [`docs/constant-time.md`](docs/constant-time.md) for the exact constant-time @@ -11,9 +12,9 @@ secret-bearing type and heap inventory, evidence, [`docs/compliance.md`](docs/compliance.md) for regulatory positioning, and [`ct.toml`](ct.toml) for the machine-readable CT claim set. -## Audit Scope +## Audit scope -Review the `ct-intended` candidate core before the rest of the repository: +Review the `ct_intended` candidate core before the rest of the repository: 1. X25519 scalar multiplication. 2. Ed25519 signing and secret-key public derivation. @@ -25,11 +26,13 @@ Review the `ct-intended` candidate core before the rest of the repository: 7. MAC/tag verification, fixed-size owner comparison/declassification, and selected password-verification comparisons. -Public parsing, raw hashes, checksums, non-cryptographic hashes, public-key -verification math, benchmark paths, unlisted targets, unlisted feature sets, and -unlisted build configurations are not blanket constant-time claims. +This order prioritizes secret-dependent computation; it does not remove public +parsers, dispatch, or unsafe kernels from the security boundary. Public parsing, +raw hashes, checksums, non-cryptographic hashes, public-key verification math, +benchmark paths, and unlisted build configurations carry no blanket +constant-time claim. -## System Boundary +## System boundary `rscrypto` is a primitives library. It computes hashes, MACs, KDFs, password hashes, AEADs, signatures, key exchanges, and checksums on caller-provided @@ -37,19 +40,19 @@ inputs. It does not open sockets, read the clock, or spawn production threads outside the opt-in `parallel` feature. With `std`, runtime CPU detection may read OS-exposed capability data such as `/proc/self/auxv`, `/proc/cpuinfo`, and sysfs; `getrandom` constructors obtain randomness from the operating system. -The crate does not read application data or manage keys on disk. Protocol -design, key storage, key rotation, and transport are the caller's -responsibility. +The crate does not read application data or manage keys on disk. The caller +owns protocol design, key storage and rotation, entropy policy, nonce +lifecycle, transport, and access control. -Everything that crosses the boundary: +Inputs crossing the boundary: -| Input | Source | Trust | +| Input | Source | Assumption | |---|---|---| -| Keys, passwords, seeds | Caller | Trusted for secrecy, not validity. Individual imports enforce their documented shape and algorithm constraints. | +| Keys, passwords, seeds | Caller | The caller protects confidentiality and supplies the required entropy. Imports enforce documented shape and algorithm constraints, not key quality. | | Messages, AAD, ciphertexts, tags, signatures, encoded keys | Caller, usually relayed from a network peer | Untrusted. | -| Randomness | `getrandom` or caller-supplied fill closures | Trusted for entropy quality. Output lengths are fixed by the API. | -| CPU capability reports | CPUID, auxv, sysctl, OS APIs | Trusted. Forced-backend overrides are validated before use. | -| Build configuration | Cargo features, target features | Trusted. | +| Randomness | `getrandom` or caller-supplied fill closures | The operating system or caller provides the required entropy quality. Output lengths are fixed by the API. | +| CPU capability reports | CPUID, auxv, sysctl, OS APIs | The host reports capabilities correctly. Forced-backend overrides are validated before use. | +| Build configuration | Cargo features, target features | The builder selects and records the intended configuration. | Outputs are digests, tags, ciphertexts, signatures, derived keys, and opaque errors. Direct comparison of fixed-size secret-bearing owners returns an opaque @@ -67,9 +70,7 @@ claims remain limited to the release-evidenced configurations. 4. Plaintext inside AEAD seal and open calls. 5. Integrity of the published crate artifacts. -## Adversaries - -In scope: +## Threats in scope 1. **Network attacker.** Supplies malformed ciphertexts, signatures, tags, and encoded keys. Goals: memory corruption, reachable panics, oracle behavior @@ -80,14 +81,14 @@ In scope: configuration enters the release claim only when every required gate passes in the matching attested release bundle, under the model in [`docs/constant-time.md`](docs/constant-time.md). -3. **Caller mistakes.** Nonce reuse, dropped verification results, weak +3. **Caller misuse.** Reuses nonces, drops verification results, or selects weak parameters. The API uses typed keys and nonces, `#[must_use]` verification results, `NonceCounter` invocation budgets, opaque errors, and explicit drop cleanup for the named secret owners. 4. **Supply-chain attacker.** Targets the path between this repository and the artifact a downstream build consumes. -Out of scope: +The following threats are out of scope: - Physical side channels: power, electromagnetic, acoustic, fault injection, rowhammer. @@ -97,7 +98,7 @@ Out of scope: - Entropy failure in the OS or in caller-supplied randomness. - Protocol composition errors in downstream code. -## Attack Surface +## Attack surface Ordered by exposure to untrusted input: @@ -109,7 +110,7 @@ Ordered by exposure to untrusted input: | `unsafe` low-level code | SIMD/assembly kernels, raw buffer helpers, zeroization, and dispatch | Undefined behavior, divergence from the portable authority | | Dispatch | `src/platform`, `src/backend` | Selecting a kernel the CPU cannot run, or one that produces wrong output | -## Mitigations And Evidence +## Mitigations and evidence | Risk | Mitigation | Evidence | |---|---|---| @@ -119,9 +120,9 @@ Ordered by exposure to untrusted input: | Timing leakage | Constant-time coding rules on claimed paths | `ct.toml` evidence gate: timing tests, generated-code review, binary checks where supported | | Oracle behavior | Opaque errors, failed-open output clearing, single-bit failure shape | AEAD and verification tests, fuzz targets | | Secret exposure at rest | Zeroize at the last owned use and on drop, masked `Debug` and errors, and sealed fixed-size comparison only on semantic secret owners | [`docs/secret-ownership.md`](docs/secret-ownership.md), [`docs/secret-lifecycle.md`](docs/secret-lifecycle.md), `scripts/check/zeroize-evidence.sh`, and `tests/secret_redaction.rs` | -| Supply chain | Minimal optional runtime dependencies, `cargo deny`, `cargo audit`, signed tags, Trusted Publishing, release attestations | `deny.toml`, `.github/workflows/release.yaml`, `docs/release.md` | +| Supply chain | Minimal optional runtime dependencies, `cargo deny`, `cargo audit`, signed tags, Trusted Publishing, release attestations | [`deny.toml`](deny.toml), [`.github/workflows/release.yaml`](.github/workflows/release.yaml), [`docs/release.md`](docs/release.md) | -## Known Gaps +## Known gaps - No third-party security audit is claimed. - Named secret owners and explicit temporaries use volatile source-level wipes, @@ -137,9 +138,9 @@ Ordered by exposure to untrusted input: claim. Windows, Linux MUSL, Intel macOS, bare-metal, and WASM physical timing evidence remains explicitly deferred. -## Review Priorities +## Review priorities -Where an external review buys the most, in order: +Prioritize external review in this order: 1. The candidate constant-time core listed above. 2. RSA DER import and the PKCS#1 v1.5, PSS, and OAEP padding checks. diff --git a/benches/auth.rs b/benches/auth.rs index 2272e19c..9a37ac57 100644 --- a/benches/auth.rs +++ b/benches/auth.rs @@ -147,10 +147,11 @@ fn hmac_sha256(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("rustcrypto", len), data, |b, d| { + let base_mac = RustCryptoHmacSha256::new_from_slice(&key).unwrap(); b.iter(|| { use hmac::Mac as _; - let mut mac = RustCryptoHmacSha256::new_from_slice(black_box(&key)).unwrap(); + let mut mac = base_mac.clone(); mac.update(black_box(d)); black_box(mac.finalize().into_bytes()) }) @@ -1114,7 +1115,7 @@ fn ed25519_sign(c: &mut Criterion) { let secret_bytes = [9u8; 32]; let secret = Ed25519SecretKey::from_bytes(secret_bytes); - let keypair = Ed25519Keypair::from_secret_key(secret); + let keypair = Ed25519Keypair::from_secret_key(secret.duplicate_secret()); let signing_key = SigningKey::from_bytes(&secret_bytes); aws_lc_bench! { let aws_kp = aws_lc_rs::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes).unwrap(); @@ -1131,6 +1132,10 @@ fn ed25519_sign(c: &mut Criterion) { for (len, data) in &inputs { common::set_throughput(&mut g, *len); + g.bench_with_input(BenchmarkId::new("rscrypto-direct-secret", len), data, |b, d| { + b.iter(|| black_box(black_box(&secret).sign(black_box(d)))) + }); + g.bench_with_input(BenchmarkId::new("rscrypto", len), data, |b, d| { b.iter(|| black_box(black_box(&keypair).sign(black_box(d)))) }); diff --git a/benches/password_hashing.rs b/benches/password_hashing.rs index bacbd57a..c6037893 100644 --- a/benches/password_hashing.rs +++ b/benches/password_hashing.rs @@ -320,17 +320,10 @@ fn argon2id_phc_roundtrip(_c: &mut Criterion) { // Stub when the PHC feature is disabled. } -/// CI-friendly lane parallelism scaling curve. With `parallel` enabled, -/// `p > 1` triggers the `rayon::scope`-driven slice driver; `p == 1` -/// takes the fast-path that skips rayon entirely. Same memory cost -/// across every `p`, so the per-iteration work is identical (Argon2's -/// matrix size depends on `m`, not `p`); only the scheduling differs. -/// -/// At 4 MiB the rayon task-dispatch overhead per slice is a measurable -/// fraction of the per-lane compute, which caps the observed efficiency -/// around 50 % at `p=4`. The overhead-to-work ratio improves with -/// larger matrices — see [`argon2id_parallel_owasp`] for the realistic -/// deployment-scale scaling curve. +/// CI-friendly lane-parallel scaling curve. With `parallel` enabled, +/// `p > 1` uses the `rayon::scope` slice driver; `p == 1` skips Rayon. +/// Every row holds total memory and time cost constant while varying the +/// lane count. #[cfg(feature = "parallel")] fn argon2id_parallel_scaling(c: &mut Criterion) { let mut g = c.benchmark_group("argon2id-parallel"); @@ -362,11 +355,8 @@ fn argon2id_parallel_scaling(c: &mut Criterion) { g.finish(); } -/// OWASP-scale lane parallelism scaling curve. Memory budget large -/// enough that the rayon task-dispatch overhead fades into noise and -/// the measurement reflects true compute parallelism. Sample size and -/// measurement window match [`argon2id_owasp`] so the group stays -/// comparable. +/// OWASP-scale lane-parallel scaling curve. Sample size and measurement +/// window match [`argon2id_owasp`] so the raw rows are comparable. #[cfg(feature = "parallel")] fn argon2id_parallel_owasp(c: &mut Criterion) { let mut g = c.benchmark_group("argon2id-parallel-owasp"); diff --git a/benchmark_results/OVERVIEW.md b/benchmark_results/OVERVIEW.md index 08105791..a7fad017 100644 --- a/benchmark_results/OVERVIEW.md +++ b/benchmark_results/OVERVIEW.md @@ -6,6 +6,7 @@ Sources: - Linux commit: `596498f0e07e869eac71fd31c157aa1b22186239`. - Linux artifacts: nine successful `benchmark-*` artifacts extracted into `benchmark_results/2026-07-04/linux/*/results.txt`. - Local macOS run: `benchmark_results/2026-07-04/macos/aarch64/results.txt` at commit `596498f0e07e869eac71fd31c157aa1b22186239`. +- Local Ed25519 direct-secret before/after diagnostic, recorded below. Scope: the 2026-07-04 nine-runner Linux CI benchmark matrix for commit `596498f`. Ratios are `external_crate_time / rscrypto_time`; higher is better. Wins are `>1.05x`, ties are `0.95x..1.05x`, and losses are `<0.95x`. Fastest-external comparisons keep only the fastest external implementation for each platform, primitive, operation, and input shape. Internal kernel, scratch-buffer, padding-only, cold-path, PHC roundtrip, parallel-scaling, threshold-selection, public-overhead, and phase-attribution microbenches are parsed as raw rows but excluded from external win/loss claims. The macOS local run is listed separately and is not mixed into Linux CI claims. @@ -13,8 +14,48 @@ This is a historical snapshot of commit `596498f`, not an inventory of the current public API. Primitive rows remain as measured even when a later commit changes or removes that surface. +Equivalence correction: the historical RustCrypto HMAC-SHA-256 rows include +key setup inside the timed loop, while rscrypto, ring, and AWS-LC use reusable +keyed state. Consequently, the HMAC-SHA-256 row and aggregate statistics that +include it are archival measurements, not equivalent-work performance claims. +The current benchmark source precomputes the RustCrypto state; publish new +HMAC-SHA-256 and aggregate ratios only after a complete benchmark artifact is +regenerated. + Coverage note: this is a full Linux CI public benchmark pass. It includes checksum, hash, XOF, MAC, KDF, password-hashing, BLAKE2/BLAKE3, RSA import/verification, ECDSA P-256/P-384 signing and verification, Ed25519, X25519, AEAD, and ML-KEM-512/768/1024 keygen, encapsulation, and decapsulation rows. ML-KEM phase/arithmetic microbenches are present in the raw artifacts and intentionally excluded from release-level competitor claims. +## 2026-07-28 Ed25519 Direct-Secret Diagnostic + +This local diagnostic compares the exact 1 KiB +`ed25519/sign/rscrypto-direct-secret/1024` Criterion case before and after the +maintenance remediation that removed duplicate secret expansion. The baseline +source is repository commit `c7338116bf8155566f9a028db1b28b5f0665e370` +with only the identical benchmark row added. The current source is that commit +plus the maintenance working-tree diff. + +Both runs used the pinned `rustc 1.97.0-nightly (ca9a134e0 2026-04-26)` +toolchain on the same Apple Silicon macOS host. Criterion used 50 samples, a +1-second warm-up, and a 3-second measurement window. + +| Source | Median | 95% confidence interval | Mean | +|---|---:|---:|---:| +| Baseline | 21.892 µs | 21.874–21.919 µs | 21.885 µs | +| Current | 21.754 µs | 21.704–21.799 µs | 21.757 µs | + +The observed current/baseline median ratio is 0.9937. This check found no +regression. It was not an interleaved release benchmark, so it does not support +a speedup claim. + +The repository policy retains only this curated overview. The local Criterion +metadata, estimates, and raw 50-sample files were distinct and hashed before +curation: + +| Artifact | Baseline SHA-256 | Current SHA-256 | +|---|---|---| +| `benchmark.json` | `6d27e19fd2a9563ecea5328345420c12b79f9924d3ecde179bc0166f5a62e6dd` | `6d27e19fd2a9563ecea5328345420c12b79f9924d3ecde179bc0166f5a62e6dd` | +| `estimates.json` | `728945652c3ec804ec064e9888fc431a5fa3528e885edf76e350392ae95ea2fc` | `3b987405f949847972740cb549826d46f2529caa1187bc13786f7d662ca63e03` | +| `sample.json` | `f36052bcf65362d6203a6be768e251822dc3182ce8fab75dd9bba20097db30f9` | `a92a7e9fcc1af048c2bb5dcfd8782d07b8727e46477a27bc7948cd02c7a8a6bc` | + ## Headline | Scope | Pairs | W/T/L | Win % | Geomean | Median | diff --git a/ct.toml b/ct.toml index dac497a5..ebcf5be8 100644 --- a/ct.toml +++ b/ct.toml @@ -46,23 +46,121 @@ formal_limitation = "BINSEC owner kernels cover the representative 16-, 32-, 48- downstream_limitation = "Evidence binds this final linked harness executable and its exact compiler/linker configuration. It does not claim constant-time behavior for arbitrary downstream binaries that rebuild or inline rscrypto differently. Production owner comparisons return an opaque CtDecision and the evidence ABI declassifies only at its retained entrypoints." [[asm_public_operand]] -primitive = "password.argon2i" -root = "ct_entry_argon2i_verify" +primitives = ["password.argon2i"] +roots = ["ct_entry_argon2i_verify"] symbol = "rscrypto::auth::argon2::argon2_hash_with_kernel_inner" kind = "variable_latency_division" -max_count = 2 +max_count = 3 source = "src/auth/argon2/mod.rs:1751" -rationale = "The divisions derive matrix dimensions and iteration bounds from validated public Argon2Params and the caller-visible output length. Password and salt contents do not feed the divisors." +rationale = "The divisions derive matrix dimensions and iteration bounds from validated public Argon2Params, the caller-visible output length, and the target's public vector length. Password and salt contents do not feed the divisors." [[asm_public_operand]] -primitive = "password.argon2i" -root = "ct_entry_argon2i_verify" +primitives = ["password.argon2i"] +roots = ["ct_entry_argon2i_verify"] symbol = "rscrypto::auth::argon2::fill_segment_inner" kind = "variable_latency_division" max_count = 2 source = "src/auth/argon2/mod.rs:1434" rationale = "For the Argon2i retained root, address generation and segment positions are data-independent; these divisions use public lane, segment, and matrix dimensions rather than password-derived contents." +[[asm_public_operand]] +primitives = ["mac.kmac256_verify"] +roots = ["ct_entry_kmac256_verify"] +symbol = "::verify_tag" +kind = "variable_latency_division" +max_count = 1 +source = "src/auth/kmac.rs:149" +rationale = "LLVM's scalable-vector loop partition divides the caller-visible expected-tag length by the target's public vector length. Tag contents do not feed the operands." + +[[asm_public_operand]] +primitives = ["rsa.private_key_material", "rsa.private_ops"] +roots = [ + "ct_entry_rsa_oaep_decrypt_fixed_blinding", + "ct_entry_rsa_pkcs1v15_decrypt_fixed_blinding", + "ct_entry_rsa_pkcs1v15_sign_fixed_blinding", + "ct_entry_rsa_private_key_pkcs8_roundtrip", + "ct_entry_rsa_pss_sign_fixed_blinding", +] +symbol = "rscrypto::auth::rsa::mont_mul_cios_portable" +kind = "variable_latency_division" +max_count = 1 +source = "src/auth/rsa.rs:10000" +rationale = "LLVM's scalable-vector loop partition divides the public modulus-width iteration count by the target's public vector length. Private limb contents do not feed the operands." + +[[asm_public_operand]] +primitives = ["rsa.private_key_material", "rsa.private_ops"] +roots = [ + "ct_entry_rsa_oaep_decrypt_fixed_blinding", + "ct_entry_rsa_pkcs1v15_decrypt_fixed_blinding", + "ct_entry_rsa_pkcs1v15_sign_fixed_blinding", + "ct_entry_rsa_private_key_pkcs8_roundtrip", + "ct_entry_rsa_pss_sign_fixed_blinding", +] +symbol = "rscrypto::auth::rsa::mont_reduce_cios" +kind = "variable_latency_division" +max_count = 1 +source = "src/auth/rsa.rs:10049" +rationale = "LLVM's scalable-vector loop partition divides the public modulus-width iteration count by the target's public vector length. Private limb contents do not feed the operands." + +[[asm_public_operand]] +primitives = ["rsa.private_key_material", "rsa.private_ops"] +roots = [ + "ct_entry_rsa_oaep_decrypt_fixed_blinding", + "ct_entry_rsa_pkcs1v15_decrypt_fixed_blinding", + "ct_entry_rsa_pkcs1v15_sign_fixed_blinding", + "ct_entry_rsa_private_key_pkcs8_roundtrip", + "ct_entry_rsa_pss_sign_fixed_blinding", +] +symbol = "rscrypto::auth::rsa::private_import_is_probable_prime" +kind = "variable_latency_division" +max_count = 2 +source = "src/auth/rsa.rs:6877" +rationale = "LLVM's scalable-vector loop partitions divide the caller-visible private-component byte width and its derived fixed limb width by the target's public vector length. Candidate contents do not feed the operands." + +[[asm_public_operand]] +primitives = ["rsa.private_ops"] +roots = [ + "ct_entry_rsa_oaep_decrypt_fixed_blinding", + "ct_entry_rsa_pkcs1v15_decrypt_fixed_blinding", + "ct_entry_rsa_pkcs1v15_sign_fixed_blinding", + "ct_entry_rsa_pss_sign_fixed_blinding", +] +symbol = "rscrypto::auth::rsa::private_import_unsigned_be_mod_to_fixed" +kind = "variable_latency_division" +max_count = 1 +source = "src/auth/rsa.rs:9499" +rationale = "LLVM's scalable-vector loop partition divides the public modulus limb width by the target's public vector length. The private representative contents do not feed the operands." + +[[asm_public_operand]] +primitives = ["rsa.private_key_material", "rsa.private_ops"] +roots = [ + "ct_entry_rsa_oaep_decrypt_fixed_blinding", + "ct_entry_rsa_pkcs1v15_decrypt_fixed_blinding", + "ct_entry_rsa_pkcs1v15_sign_fixed_blinding", + "ct_entry_rsa_private_key_pkcs8_roundtrip", + "ct_entry_rsa_pss_sign_fixed_blinding", +] +symbol = "rscrypto::auth::rsa::private_import_unsigned_be_mod_to_len" +kind = "variable_latency_division" +max_count = 1 +source = "src/auth/rsa.rs:9481" +rationale = "LLVM's scalable-vector loop partition divides the public component/modulus byte width by the target's public vector length. Private component contents do not feed the operands." + +[[asm_public_operand]] +primitives = ["rsa.private_key_material", "rsa.private_ops"] +roots = [ + "ct_entry_rsa_oaep_decrypt_fixed_blinding", + "ct_entry_rsa_pkcs1v15_decrypt_fixed_blinding", + "ct_entry_rsa_pkcs1v15_sign_fixed_blinding", + "ct_entry_rsa_private_key_pkcs8_roundtrip", + "ct_entry_rsa_pss_sign_fixed_blinding", +] +symbol = "rscrypto::auth::rsa::public_montgomery_r2_into" +kind = "variable_latency_division" +max_count = 1 +source = "src/auth/rsa.rs:9597" +rationale = "LLVM's scalable-vector loop partition divides the public modulus limb width by the target's public vector length. No private value feeds the operands." + [operation_inventory] schema_version = 1 authority = "ct.toml" @@ -82,10 +180,10 @@ compiler_api_prefixes = [ id = "kmac.verify_expected_output" source = "src/auth/kmac.rs" call_count = 1 -operations = ["kmac.variable_output_and_verify"] +operations = ["kmac.variable_output_and_verify", "kmac128.variable_output_and_verify"] public_length = "The caller selects KMAC output length and supplies an expected buffer of that public length." secret_contents = "The computed KMAC output and expected bytes are traversed fully for each public-length chunk." -tests = "KMAC NIST/Wycheproof/differential tests plus kmac256_valid_vs_invalid_tag DudeCT." +tests = "KMAC128/256 NIST and differential tests, KMAC256 Wycheproof tests, and kmac256_valid_vs_invalid_tag DudeCT." evidence_symbols = ["ct_entry_kmac256_verify"] evidenced_call_count = 1 limited_call_count = 0 @@ -1009,6 +1107,16 @@ right_class = "random same-length message" samples = 128 smoke_samples = 128 +[[dudect_case]] +name = "rsa_pkcs1v15_full_width_vs_short_canonical_crt_exponent" +primitive = "rsa.private_ops" +filter = "rsa_pkcs1v15_full_width_vs_short_canonical_crt_exponent" +left_class = "valid RSA-2048 key with 128-byte canonical CRT exponents" +right_class = "valid same-factor-width RSA-2048 key with a 127-byte canonical dQ" +samples = 512 +smoke_samples = 128 +notes = "Both imported keys retain 128-byte CRT exponent buffers. The whole-operation comparison also varies secret key values, as required by the private-operation CT boundary." + [[dudect_case]] name = "rsa_oaep_decrypt_fixed_vs_random_plaintext" primitive = "rsa.private_ops" @@ -1684,7 +1792,7 @@ binsec = ["aead.symmetric_transform.ascon_aead128_tag_portable.all"] id = "rsa.private_ops.pkcs1v15_sign" primitive = "rsa.private_ops" variant = "Pkcs1v15Sign" -dudect = ["rsa_pkcs1v15_fixed_vs_random_message"] +dudect = ["rsa_pkcs1v15_fixed_vs_random_message", "rsa_pkcs1v15_full_width_vs_short_canonical_crt_exponent"] binsec = ["rsa.private_ops.window_select.portable.all"] [[evidence_unit]] @@ -2591,8 +2699,8 @@ name = "x86_64-unknown-linux-gnu" group = "linux" backend = "llvm" linker = "platform-default-unpinned" -compiler_api_item_count = 2261 -compiler_api_sha256 = "a08f563d15c951a2286451972cbc524d69100bfe6e3c027d8dbed381d869767f" +compiler_api_item_count = 2221 +compiler_api_sha256 = "7a10185323957d7666b51b29cfab3969036f5551b2d5bd8345a8c1830610aed6" claim = "ct-intended" physical_timing = "required" binsec = "required" @@ -2603,8 +2711,8 @@ name = "aarch64-unknown-linux-gnu" group = "linux" backend = "llvm" linker = "platform-default-unpinned" -compiler_api_item_count = 2256 -compiler_api_sha256 = "58d9a255a6c13f093a5a0946be4c6cf3c08125415738d9e2013157cbbb84e186" +compiler_api_item_count = 2215 +compiler_api_sha256 = "44d494189799045ed6e0ffcb3e2c7453868d62f39f9a069c31ede5d28a65f7b5" claim = "ct-intended" physical_timing = "required" binsec = "required" @@ -2663,8 +2771,8 @@ name = "aarch64-apple-darwin" group = "macos" backend = "llvm" linker = "apple-ld-unpinned" -compiler_api_item_count = 2256 -compiler_api_sha256 = "58d9a255a6c13f093a5a0946be4c6cf3c08125415738d9e2013157cbbb84e186" +compiler_api_item_count = 2215 +compiler_api_sha256 = "44d494189799045ed6e0ffcb3e2c7453868d62f39f9a069c31ede5d28a65f7b5" claim = "ct-intended" physical_timing = "required" binsec = "unsupported" @@ -2688,8 +2796,8 @@ name = "s390x-unknown-linux-gnu" group = "ibm" backend = "llvm" linker = "platform-default-unpinned" -compiler_api_item_count = 2250 -compiler_api_sha256 = "613c917f01b4ac52fa19f4a3c33f1e73e154d8ce74622f98c3ab74448072b6f8" +compiler_api_item_count = 2214 +compiler_api_sha256 = "1f23434a153550dc9ada6bff47a9c7c89adc1763629827e940f4ad69e3be074c" claim = "ct-intended" physical_timing = "required" binsec = "unsupported" @@ -2701,8 +2809,8 @@ name = "powerpc64le-unknown-linux-gnu" group = "ibm" backend = "llvm" linker = "platform-default-unpinned" -compiler_api_item_count = 2250 -compiler_api_sha256 = "9f6f605b4ad27fe7fcc4de4b830ae99ee4b1ab54e5ccf848da6845c6ecc9a788" +compiler_api_item_count = 2214 +compiler_api_sha256 = "37793296770a82d75102f92a8630a9b3f9d90b59de36939ff3505554d6f66b92" claim = "ct-intended" physical_timing = "required" binsec = "unsupported" @@ -2714,8 +2822,8 @@ name = "riscv64gc-unknown-linux-gnu" group = "linux" backend = "llvm" linker = "platform-default-unpinned" -compiler_api_item_count = 2252 -compiler_api_sha256 = "7b5a444acb4147d30322f21acde5ccf1a671b13420ccbb4820e004fd118d8fb5" +compiler_api_item_count = 2214 +compiler_api_sha256 = "8f5377e62133152ba74f5623313079a862b2632fe69efd8a7535d69d2b0aef78" claim = "ct-intended" physical_timing = "required" binsec = "unsupported" @@ -2909,27 +3017,40 @@ limitation = "Nonces, encoded public values, and error metadata use ordinary equ [[operation]] id = "mac.compute_and_finalize" api = [ - "rscrypto::Mac::{new,update,update_vectored,update_io_slices,finalize,reset,mac,finalize_to_vec,mac_to_vec}", - "rscrypto::{HmacSha*,Kmac*,Poly1305}::{new,update,finalize,reset,mac}", + "rscrypto::{HmacSha256,HmacSha384,HmacSha512}::{new,update,finalize,reset,mac}", + "rscrypto::Kmac256::{new,update,finalize_into,reset,mac_into,mac_array}", ] -features = ["hmac", "hmac-sha3", "kmac", "poly1305", "alloc/std for Vec/IoSlice helpers"] +features = ["hmac", "kmac"] targets = ["all-supported", "backend selected by public target capabilities"] secret_inputs = ["MAC key", "MAC state", "message when caller classifies it secret", "computed tag"] public_inputs = ["message length", "algorithm", "backend capabilities"] -variable_time_components = ["public message-length loops", "public backend dispatch", "allocation for Vec helpers"] +variable_time_components = ["public message-length loops", "public backend dispatch"] permitted_leakage = ["algorithm", "message length", "backend", "returned tag"] claim = "ct-intended" evidence = ["primitive:mac.hmac_verify", "primitive:mac.kmac256_verify", "harness:ct_entry_hmac_sha256_verify", "harness:ct_entry_kmac256_verify"] -limitation = "Mac is implementable downstream. Each implementation owns verify; the default verify_tag delegates to it, so downstream verification timing remains outside rscrypto's evidence." +limitation = "The HMAC verification evidence computes and finalizes HMAC-SHA-2; the KMAC evidence covers KMAC256. Other MAC variants and downstream Mac implementations are inventoried separately without generated-code evidence." + +[[operation]] +id = "mac.compute_and_finalize_source_only" +api = [ + "rscrypto::Mac::{new,update,update_vectored,update_io_slices,finalize,reset,mac,finalize_to_vec,mac_to_vec}", + "rscrypto::{HmacSha3_224,HmacSha3_256,HmacSha3_384,HmacSha3_512,Kmac128,Poly1305}::{new,update,finalize,reset,mac}", +] +features = ["hmac-sha3", "kmac", "poly1305", "alloc/std for Vec/IoSlice helpers"] +targets = ["all-supported", "backend selected by public target capabilities"] +secret_inputs = ["MAC key", "MAC state", "message when caller classifies it secret", "computed tag"] +public_inputs = ["message length", "algorithm", "backend capabilities"] +variable_time_components = ["public message-length loops", "public backend dispatch", "allocation for Vec helpers"] +permitted_leakage = ["algorithm", "message length", "backend", "returned tag"] +claim = "best-effort" +limitation = "No generated-code unit currently covers HMAC-SHA-3, KMAC128, standalone Poly1305, allocating trait helpers, or downstream Mac implementations. This row records source intent only." [[operation]] id = "mac.verify_typed_tag" api = [ - "rscrypto::Mac::{verify,verify_tag}", - "rscrypto::HmacSha*::verify_tag", - "rscrypto::Poly1305::verify", + "rscrypto::{HmacSha256,HmacSha384,HmacSha512}::{verify,verify_tag}", ] -features = ["hmac", "hmac-sha3", "poly1305"] +features = ["hmac"] targets = ["all-supported"] secret_inputs = ["MAC key/state", "computed tag"] public_inputs = ["message", "typed expected tag", "public lengths"] @@ -2937,20 +3058,48 @@ variable_time_components = ["MAC computation over public length", "sealed fixed- permitted_leakage = ["message length", "tag size", "opaque verification result"] claim = "ct-intended" evidence = ["primitive:mac.hmac_verify", "unit:mac.hmac_verify.sha256", "unit:mac.hmac_verify.sha384", "unit:mac.hmac_verify.sha512"] -limitation = "The public Mac trait requires each implementation to define verify. Built-in implementations use sealed tag-owner comparisons; downstream implementations are an explicit evidence boundary." +limitation = "Evidence is variant-specific to the three built-in HMAC-SHA-2 implementations." + +[[operation]] +id = "mac.verify_typed_tag_source_only" +api = [ + "rscrypto::Mac::{verify,verify_tag}", + "rscrypto::{HmacSha3_224,HmacSha3_256,HmacSha3_384,HmacSha3_512}::{verify,verify_tag}", + "rscrypto::Poly1305::verify", +] +features = ["hmac-sha3", "poly1305"] +targets = ["all-supported"] +secret_inputs = ["MAC key/state", "computed tag"] +public_inputs = ["message", "typed expected tag", "public lengths"] +variable_time_components = ["MAC computation over public length", "sealed fixed-owner comparison", "opaque result conversion"] +permitted_leakage = ["message length", "tag size", "opaque verification result"] +claim = "best-effort" +limitation = "No generated-code unit currently covers the HMAC-SHA-3 or standalone Poly1305 verification paths. Downstream Mac implementations remain an explicit evidence boundary." [[operation]] id = "kmac.variable_output_and_verify" -api = ["rscrypto::{Kmac128,Kmac256}::{new,update,finalize,mac,mac_array,verify,verify_tag}"] +api = ["rscrypto::Kmac256::{new,update,finalize_into,mac_into,mac_array,verify,verify_tag,verify_primitive,verify_tag_primitive}"] features = ["kmac", "alloc for allocating outputs"] targets = ["all-supported"] secret_inputs = ["KMAC key", "state", "computed output"] -public_inputs = ["customization", "message", "requested output length", "expected output length"] -variable_time_components = ["public-length sponge absorption/squeeze", "public expected-length mismatch", "public-length full-content comparison"] -permitted_leakage = ["customization", "message/output lengths", "opaque verification result"] +public_inputs = ["customization", "message", "requested output length", "expected output length", "authentication minimum policy"] +variable_time_components = ["public-length sponge absorption/squeeze", "public authentication-length policy rejection", "public-length full-content comparison"] +permitted_leakage = ["customization", "message/output lengths", "authentication-length policy decision", "opaque verification result"] claim = "ct-intended" evidence = ["primitive:mac.kmac256_verify", "harness:ct_entry_kmac256_verify", "dudect:kmac256_valid_vs_invalid_tag"] -limitation = "KMAC has protocol-defined variable output. Length is explicitly classified public; this is the only MAC verification family retaining an internal public-length comparison." +limitation = "KMAC has protocol-defined variable output. The default authentication verifier rejects tags below 32 bytes; explicit primitive verification permits shorter nonempty protocol outputs. Length and the policy decision are public." + +[[operation]] +id = "kmac128.variable_output_and_verify" +api = ["rscrypto::Kmac128::{new,update,finalize_into,mac_into,mac_array,verify,verify_tag,verify_primitive,verify_tag_primitive}"] +features = ["kmac", "alloc for allocating outputs"] +targets = ["all-supported"] +secret_inputs = ["KMAC key", "state", "computed output"] +public_inputs = ["customization", "message", "requested output length", "expected output length", "authentication minimum policy"] +variable_time_components = ["public-length sponge absorption/squeeze", "public authentication-length policy rejection", "public-length full-content comparison"] +permitted_leakage = ["customization", "message/output lengths", "authentication-length policy decision", "opaque verification result"] +claim = "best-effort" +limitation = "The source follows the same structure as KMAC256 and the default verifier rejects tags below 16 bytes, but no KMAC128 generated-code evidence unit currently exists." [[operation]] id = "keyed_hash.compute_and_verify" @@ -2971,7 +3120,7 @@ limitation = "Keyed outputs are explicitly extractable/serializable. Only Blake3 [[operation]] id = "kdf.hkdf_extract_expand" -api = ["rscrypto::HkdfSha{256,384,512}::{new,extract,expand,expand_array,derive,derive_array,prk}"] +api = ["rscrypto::HkdfSha{256,384}::{new,extract,expand,expand_array,derive,derive_array,prk}"] features = ["hkdf", "hmac"] targets = ["all-supported"] secret_inputs = ["input key material", "PRK", "OKM"] @@ -2982,18 +3131,38 @@ claim = "ct-intended" evidence = ["primitive:kdf.hkdf", "unit:kdf.hkdf.sha256", "unit:kdf.hkdf.sha384"] limitation = "prk() and returned/output buffers expose derived secret material to the caller by design." +[[operation]] +id = "kdf.hkdf_sha512_extract_expand" +api = ["rscrypto::HkdfSha512::{new,extract,expand,expand_array,derive,derive_array,prk}"] +features = ["hkdf", "hmac"] +targets = ["all-supported"] +secret_inputs = ["input key material", "PRK", "OKM"] +public_inputs = ["salt unless protocol classifies it secret", "info", "requested output length"] +variable_time_components = ["public input/output-length loops", "public output-length validation"] +permitted_leakage = ["salt/info/output lengths", "length error", "explicit PRK/OKM exposure"] +claim = "best-effort" +limitation = "HKDF-SHA512 has source, vector, differential, and optimized zeroization coverage, but no constant-time generated-code evidence unit. prk() and returned/output buffers expose derived secret material by design." + [[operation]] id = "kdf.pbkdf2_derive_and_verify" -api = ["rscrypto::Pbkdf2Sha{256,512}::{derive,verify,verify_password}", "rscrypto::Pbkdf2Params::*", "rscrypto::Pbkdf2VerifyPolicy::*"] +api = [ + "rscrypto::Pbkdf2Sha{256,512}::{new,params,params_with_policy,params_with_policy_bounded}", + "rscrypto::Pbkdf2Sha{256,512}::{derive,derive_with_params,derive_array,derive_array_with_params}", + "rscrypto::Pbkdf2Sha{256,512}::{derive_key,derive_key_with_params,derive_key_primitive,derive_key_array,derive_key_array_with_params,derive_key_array_primitive}", + "rscrypto::Pbkdf2Sha{256,512}::{verify,verify_with_policy,verify_with_policy_bounded,verify_primitive}", + "rscrypto::Pbkdf2Sha{256,512}::{verify_password,verify_password_with_policy,verify_password_with_policy_bounded,verify_password_primitive}", + "rscrypto::Pbkdf2Params::*", + "rscrypto::Pbkdf2VerifyPolicy::*", +] features = ["pbkdf2", "alloc", "phc-strings"] targets = ["all-supported"] secret_inputs = ["password", "derived key", "computed verifier"] -public_inputs = ["salt", "iteration count", "output length", "PHC string", "verification policy"] -variable_time_components = ["public iteration count", "public output-length loops", "PHC parsing and policy rejection", "public-length full-content comparison"] -permitted_leakage = ["parameters", "salt", "lengths", "policy decision", "opaque verification result"] +public_inputs = ["salt", "iteration count", "caller-selected maximum iteration count", "output length", "PHC string", "verification policy"] +variable_time_components = ["public iteration count", "public output-length loops", "PHC parsing and public work-policy rejection", "public-length full-content comparison"] +permitted_leakage = ["parameters", "salt", "lengths", "public work-policy decision", "opaque verification result"] claim = "ct-intended" evidence = ["primitive:kdf.pbkdf2", "unit:kdf.pbkdf2.sha256", "unit:kdf.pbkdf2.sha512"] -limitation = "Primitive verify accepts a caller-selected public output length; PHC helpers pin and approve resource bounds before deriving." +limitation = "Iteration count, policy minima, and max_iterations are public. Bounded verification rejects excessive public work before constructing password-derived HMAC state; primitive and compatibility methods deliberately do not cap it." [[operation]] id = "password.argon2i" @@ -3149,7 +3318,7 @@ variable_time_components = ["public modulus-width loops", "public padding/hash p permitted_leakage = ["modulus/profile/lengths", "opaque success/failure"] claim = "ct-intended" evidence = ["primitive:rsa.private_ops", "unit:rsa.private_ops.pkcs1v15_sign", "unit:rsa.private_ops.pss_sign", "unit:rsa.private_ops.oaep_decrypt", "unit:rsa.private_ops.pkcs1v15_decrypt"] -limitation = "All remaining variable-length internal comparisons use public modulus/profile-derived shapes. Full private operations remain target-artifact/DudeCT evidence, not source-only claims." +limitation = "Retained CRT exponents are normalized to the corresponding factor width before arithmetic, so canonical exponent encoding length is not a steady-state control or address input. Full private operations remain target-artifact/DudeCT evidence, not source-only claims." [[operation]] id = "rsa.public_operations" @@ -3582,7 +3751,7 @@ public = ["message", "ciphertext", "profile", "modulus_length", "salt_length", " may_leak = ["input_length", "public_profile", "opaque_success_or_failure", "public_rng_failure"] must_not_leak_ref = "common_secret_data" required = ["tier_a"] -notes = "Top-level OS randomness and public profile parsing may fail publicly. Steady-state private math and same-width padding/failure behavior are CT-critical." +notes = "Top-level OS randomness and public profile parsing may fail publicly. Validated CRT exponents are retained at the corresponding factor width before steady-state private arithmetic. Steady-state private math and same-width padding/failure behavior are CT-critical." [primitive.harness] status = "covered" symbols = [ diff --git a/docs/benchmarking.md b/docs/benchmarking.md index 5c08beea..f3235c0e 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -1,13 +1,19 @@ # Benchmarking -`rscrypto` publishes benchmarks so users can see where it wins, ties, or loses -in shape-compatible comparisons. +Use these benchmarks to compare a specific primitive, operation, input shape, +and target. Do not treat a crate-wide aggregate as a deployment result. Benchmark numbers are only meaningful with their platform, commit, feature set, and comparison shape. Treat every headline number as a pointer to the raw results in [`benchmark_results/`](../benchmark_results/). -## Reading The Numbers +The published 2026-07-04 aggregate is archival, not an equivalent-work +performance claim. Its RustCrypto HMAC-SHA-256 rows include key setup inside +the timed loop while the rscrypto, `ring`, and AWS-LC rows reuse keyed state. +The current benchmark source corrects that mismatch; publish a new aggregate +only after regenerating the complete artifact. + +## Read the numbers Speedup is reported as: @@ -18,10 +24,12 @@ external_crate_time / rscrypto_time Values above `1.00x` mean `rscrypto` was faster for that row. Values below `1.00x` mean the comparison crate was faster. -Use the geomean summaries for broad shape. Use individual rows when a specific -primitive or message size matters to your deployment. +The published W/T/L summaries classify ratios above `1.05x` as wins, ratios +from `0.95x` through `1.05x` as ties, and ratios below `0.95x` as losses. Use +individual equivalent-work rows—not the archival aggregate—when a primitive or +message size matters to a deployment. -## Published Sources +## Published sources Raw Criterion output lives under: @@ -38,7 +46,7 @@ Platform-specific claims need platform-specific raw results. A strong x86_64 result does not imply the same result on aarch64, Power, s390x, RISC-V, WASM, or `no_std`. -## Competitor Set +## Competitor set The comparison set in the published snapshot is Rust-focused and shape-compatible: @@ -64,10 +72,13 @@ Some common libraries are not primary benchmark baselines: `aws-lc-rs` comparison. - Generic trait crates such as `digest` are not algorithms. -## Shape Notes +## Shape notes - ECDSA rows are split by curve and operation. P-256 uses SHA-256; P-384 uses SHA-384. +- Ed25519 signing includes both retained-keypair signing and direct + `Ed25519SecretKey::sign` rows. The latter includes secret expansion and + public-key derivation on every call. - ML-KEM end-to-end rows are split by parameter set and operation: key generation, encapsulation, and decapsulation for ML-KEM-512, ML-KEM-768, and ML-KEM-1024. @@ -78,7 +89,7 @@ Some common libraries are not primary benchmark baselines: - `dryoc` XChaCha20-Poly1305 is excluded from one-shot AEAD rows because the exposed benchmark shape is libsodium secretstream, not detached one-shot AEAD. -## Reproducing Locally +## Reproduce locally Use the `just bench` recipes when you want local numbers: @@ -92,3 +103,5 @@ just bench mlkem Local runs are useful for capacity planning on your hardware. They should not be mixed with published claims unless the run metadata and raw results are kept. +On macOS, the local benchmark entry point selects the host CPU unless the caller +provides `RUSTFLAGS` or `CARGO_ENCODED_RUSTFLAGS`; normal builds remain portable. diff --git a/docs/compliance.md b/docs/compliance.md index 3d6d1fd6..45859257 100644 --- a/docs/compliance.md +++ b/docs/compliance.md @@ -1,20 +1,21 @@ # Compliance Posture -`rscrypto` can support a compliance program, but it is not compliance by -itself. It is a pure Rust cryptographic primitives crate with public source, -explicit feature flags, documented security boundaries, vector coverage, and a -scoped constant-time evidence model. +`rscrypto` is not a validated cryptographic module and does not make a system +compliant. It can supply primitive and implementation-assurance evidence inside +a separately defined module, product, deployment, and assessment boundary. Use it when your policy allows non-validated Rust primitives, or when you are -building your own module boundary and evidence package. Do not use it as a -FIPS 140-3 answer by itself. +building and validating your own module boundary. Do not present the crate +alone as a FIPS 140-3 or CMVP answer. External standards references in this page were checked against NIST CSRC on -2026-07-25. Several linked publications have active revision or errata notes; -regulated deployments must review those notes, the current standard text, and -their assessor's requirements before release. +2026-07-29. CMVP requirements include the current management manual, +implementation guidance, and SP 800-140 series—not only the top-level FIPS +140-3 publication. Several linked publications have active revision or errata +notes; regulated deployments must review those notes, the current standard +text, and their assessor's requirements before release. -## Quick Answer +## Quick answer | User question | Answer | |---|---| @@ -22,17 +23,19 @@ their assessor's requirements before release. | Can I use `rscrypto` inside a FIPS-oriented project? | Possibly, if your project owns the module boundary, operational environment, allowed algorithms, self-tests, documentation, and validation path. | | Which APIs are standards-aligned? | See the inventory below. It is an algorithm and API map, not a validation claim. | | Which evidence can I cite during review? | Start with [`constant-time.md`](constant-time.md), [`test-vector-coverage.md`](test-vector-coverage.md), [`features.md`](features.md), and [`platforms.md`](platforms.md). | -| What if procurement requires a validated cryptographic module? | Use a validated module, or validate a larger module that includes `rscrypto` under a defined boundary. | +| What if procurement requires a validated cryptographic module? | Use a module listed by CMVP for the required environment, or pursue validation of a defined module that includes `rscrypto`. | -## Standards-Aligned Primitives +## Standards-aligned primitives These surfaces are useful for NIST-oriented design reviews because the public API maps to named standards or profiles. They are not FIPS 140-3 validation -claims. +claims or a list of CMVP-approved security functions. Check current CMVP +requirements before placing any surface inside a validation boundary. | Area | `rscrypto` surface | Boundary users must still own | |---|---|---| | AES-GCM AEAD | `Aes128Gcm`, `Aes256Gcm` (`aes-gcm`) follow the SP 800-38D GCM shape. | Key lifecycle, nonce lifecycle, invocation limits, protocol binding, and allowed-use policy. | +| Ascon | `AsconAead128`, `AsconHash256`, `AsconXof128`, and `AsconCxof128` map to the SP 800-232 parameter sets. | Current CMVP eligibility, protocol profile, key and nonce lifecycle, customization strings, and validation boundary. | | SHA-2 / SHA-3 / SHAKE | `Sha224`, `Sha256`, `Sha384`, `Sha512`, `Sha512_256`, `Sha3_*`, `Shake128`, `Shake256`. | Algorithm selection, message/domain separation, and any protocol-specific hash profile. | | KMAC / cSHAKE | `Kmac128`, `Kmac256`, `Cshake128`, `Cshake256`. | Customization strings, key management, and protocol profile. | | HMAC / HKDF / PBKDF2 | `HmacSha256`, `HmacSha384`, `HmacSha512`, `HmacSha3_*`, `HkdfSha256`, `HkdfSha384`, `HkdfSha512`, `Pbkdf2Sha256`, `Pbkdf2Sha512`. | Key separation, salt/IKM policy, iteration counts, output lengths, and password policy. | @@ -43,7 +46,7 @@ claims. For exact public types and features, use [`types.md`](types.md) and [`features.md`](features.md). -## Not a FIPS-Oriented Claim +## Outside the FIPS-oriented inventory The following APIs may be correct and useful, but they should not be presented as FIPS 140-3 validated, CMVP certified, or part of the FIPS-oriented inventory @@ -52,7 +55,7 @@ above unless your own compliance target explicitly allows them: | Area | Examples | |---|---| | Misuse-resistant or non-NIST AEADs | `Aes128GcmSiv`, `Aes256GcmSiv`, `ChaCha20Poly1305`, `XChaCha20Poly1305`, `Aegis256` | -| Other hashes / XOFs | `Blake*`, `Blake3`, `Ascon*`, `Xxh3`, `RapidHash64` | +| Other hashes / XOFs | `Blake*`, `Blake3`, `Xxh3`, `RapidHash64` | | Other MACs | Standalone `Poly1305` | | Other public-key primitives | `Ed25519*`, `X25519*` | | Password hashing outside SP 800-132 | `Argon2*`, `Scrypt` | @@ -62,7 +65,7 @@ This table is about compliance positioning, not engineering quality. For example, Argon2 and scrypt are appropriate password-hashing choices in many systems, but they are not a FIPS 140-3 validation claim. -## Evidence Users Can Review +## Evidence users can review `rscrypto` publishes evidence that can help a security review or vendor-risk review, but none of it replaces an audit or validation certificate. @@ -76,7 +79,7 @@ review, but none of it replaces an audit or validation certificate. | Platform and dispatch model | [`platforms.md`](platforms.md) | | Vulnerability reporting process | [`../SECURITY.md`](../SECURITY.md) | -## What Users Still Own +## What users still own If `rscrypto` is part of a regulated system, the integrator owns: @@ -93,10 +96,11 @@ If `rscrypto` is part of a regulated system, the integrator owns: - Assessor, lab, auditor, or customer evidence requests. `portable-only` can help audit-constrained builds by forcing runtime dispatch -toward portable backends. It does not remove SIMD code from the binary, create -a constant-time proof, or create a FIPS validation boundary. +toward portable backends. It does not override compile-time target features, +remove accelerated code from the binary, create a constant-time proof, or +create a FIPS validation boundary. -## Accurate Downstream Wording +## Accurate downstream wording Use this boundary in downstream documentation: @@ -118,9 +122,10 @@ approved module drop-in compliance replacement ``` -## Primary References +## Primary references - [NIST CMVP FIPS 140-3 Standards](https://csrc.nist.gov/projects/cryptographic-module-validation-program/fips-140-3-standards) +- [NIST CMVP Management Manual and Implementation Guidance](https://csrc.nist.gov/projects/cryptographic-module-validation-program/fips-140-3-ig-announcements) - [NIST FIPS 180-4: Secure Hash Standard](https://csrc.nist.gov/pubs/fips/180-4/upd1/final) - [NIST FIPS 202: SHA-3 Standard](https://csrc.nist.gov/pubs/fips/202/final) - [NIST FIPS 203: Module-Lattice-Based Key-Encapsulation Mechanism](https://csrc.nist.gov/pubs/fips/203/final) @@ -128,3 +133,4 @@ drop-in compliance replacement - [NIST SP 800-38D: GCM and GMAC](https://csrc.nist.gov/pubs/sp/800/38/d/final) - [NIST SP 800-132: Password-Based Key Derivation](https://csrc.nist.gov/pubs/sp/800/132/final) - [NIST SP 800-185: SHA-3 Derived Functions](https://csrc.nist.gov/pubs/sp/800/185/final) +- [NIST SP 800-232: Ascon-Based Lightweight Cryptography Standards](https://csrc.nist.gov/pubs/sp/800/232/final) diff --git a/docs/constant-time.md b/docs/constant-time.md index c88f17e4..b3503a5e 100644 --- a/docs/constant-time.md +++ b/docs/constant-time.md @@ -4,12 +4,12 @@ style alone is not enough: the claim depends on the crate version, commit, compiler, target, CPU features, enabled features, and generated binary. -Unlisted configurations are not covered by a constant-time release claim. +An unlisted configuration has no constant-time release claim. The machine-readable source of truth is [`ct.toml`](../ct.toml). The sections below explain how to read that boundary. -## Claim Definition +## Claim definition For the declared secret inputs of a claimed primitive, the generated binary must not let those secrets influence: @@ -26,7 +26,7 @@ not let those secrets influence: Public inputs may still affect control flow, lengths, allocation size, backend selection, and public error handling. -## Threat Model +## Threat model The policy targets software-observable timing leakage from secret data through control flow, memory access, dispatch, allocation, failure shape, and generated @@ -40,7 +40,7 @@ Speculation is handled by avoiding secret-dependent branches and addresses in claimed code paths. This is not a blanket Spectre-class guarantee for a whole process. -## Candidate Surfaces +## Candidate surfaces `ct.toml` places the following highest-sensitivity surfaces inside the release evidence gate. This is intent, not a standalone public claim: @@ -61,8 +61,11 @@ evidence gate. This is intent, not a standalone public claim: Symmetric encryption, polynomial authenticators, KDF internals, and password-hashing internals are CT-relevant when key or password material is live. +Validated RSA keys retain each CRT exponent at its corresponding factor width +before steady-state private arithmetic. DER import and export remain +variable-shape operations outside that steady-state boundary. -## Not Blanket Claims +## Excluded unless listed The following are not constant-time claims unless a specific manifest entry says otherwise: @@ -79,7 +82,7 @@ otherwise: Public length may leak. Public algorithm/profile selection may leak. A single opaque authentication success/failure result may leak. -## Source-Level Decision Boundary +## Source-level decision boundary Secret-bearing fixed-size keys, shared secrets, authentication tags, keypairs, and keyed outputs do not implement `PartialEq` or `Eq`. Their inherent `ct_eq` @@ -92,12 +95,14 @@ Verification APIs keep that boundary inside the primitive and return one opaque `Result`. Public keys, nonces, signatures, and ciphertext containers are public data and retain ordinary equality where useful. -This API prevents accidental source-level branching during comparison. It does -not prove the generated machine code. A constant-time claim still requires the -exact compiler, target, CPU features, crate features, profile, linker, and -binary recorded by the matching release evidence. +This API removes ordinary equality and implicit Boolean conversion from the +secret-bearing owner type. Callers can still export bytes or explicitly +declassify a decision. The API boundary does not prove the generated machine +code. A constant-time claim still requires the exact compiler, target, CPU +features, crate features, profile, linker, and binary recorded by the matching +release evidence. -## Target Scope +## Target scope A target is not claimed because it builds. It is claimed only when the release has evidence for the exact compiler, codegen backend, linker, target @@ -132,20 +137,22 @@ unreviewed hand-written assembly. Linux MUSL, macOS `x86_64`, Windows MSVC, bare-metal `no_std`, and WASM builds may compile and may follow the same coding rules, but physical timing evidence -is explicitly deferred. Apple Silicon macOS evidence is local rather than part -of the release bundle. Artifact and heuristic analysis for a deferred target -must never be represented as native physical timing evidence. +is explicitly deferred. Apple Silicon macOS evidence is collected on a physical +local Mac rather than included in the release bundle; `just test-rsa-macos-asm` +records the RSA assembly equivalence and binary-presence checks. Artifact and +heuristic analysis for a deferred target must never be represented as native +physical timing evidence. `portable-only` constrains runtime dispatch to portable backends. It is useful for audit-constrained builds, but it is not a proof by itself. ## Evidence -Source inspection and `ct.toml` are not sufficient to establish a release claim. The matching signed GitHub release -must contain all of: +Source inspection and `ct.toml` do not establish a release claim. The matching +signed GitHub release must contain all of: -- The attested release manifest, source archive, crate, and `SHA256SUMS` binding the release tag, commit, toolchain, - and artifacts. +- The attested release manifest, source archive, crate, and `SHA256SUMS` + binding the release tag, commit, toolchain, and artifacts. - An attested `rscrypto-X.Y.Z-ct-evidence.tar.gz` built from the same release commit. - `CT-EVIDENCE-BUNDLE.json`, naming the version, full commit, release profile, @@ -202,8 +209,9 @@ date. Source or disassembly movement invalidates it. BINSEC is required on the GNU Linux targets supported by the workflow. Every manifest-required kernel must report `secure`. Other target reports record BINSEC as `not_applicable` with the target policy reason; that status is not -binary proof. Each formal result is bound to its hashed proof driver, -disassembly, configuration, solver log, candidate identity, and toolchain. +binary proof. Each formal result is bound to the exact BINSEC executable hash, +hashed proof driver, disassembly, configuration, solver log, candidate +identity, and toolchain. Statistical timing checks must be described precisely: @@ -213,10 +221,11 @@ No leakage detected for this configuration. They are evidence, not a formal proof. -## Consumer Verification +## Verify a release For release `vX.Y.Z`, download the crate, CT bundle, and checksums from that -exact GitHub release, then verify both attestations and hashes: +exact GitHub release. Replace `X.Y.Z` consistently, then verify the release +attestation, asset attestations, checksums, and internal CT manifest: ```bash gh release download vX.Y.Z --repo loadingalias/rscrypto \ @@ -235,8 +244,9 @@ gh attestation verify rscrypto-X.Y.Z-ct-evidence.tar.gz --repo loadingalias/rscr gh attestation verify rscrypto-X.Y.Z-repository-controls.json --repo loadingalias/rscrypto gh attestation verify rscrypto-X.Y.Z-release-manifest.json --repo loadingalias/rscrypto gh attestation verify SHA256SUMS --repo loadingalias/rscrypto -mkdir ct-evidence && tar -xzf rscrypto-X.Y.Z-ct-evidence.tar.gz -C ct-evidence -(cd ct-evidence && sha256sum --check CT-EVIDENCE-MANIFEST.txt) +ct_evidence_dir=$(mktemp -d) +tar -xzf rscrypto-X.Y.Z-ct-evidence.tar.gz -C "$ct_evidence_dir" +(cd "$ct_evidence_dir" && sha256sum --check CT-EVIDENCE-MANIFEST.txt) ``` Inspect `CT-EVIDENCE-BUNDLE.json` and use only lanes whose exact target, diff --git a/docs/features.md b/docs/features.md index 05826c4f..622c5073 100644 --- a/docs/features.md +++ b/docs/features.md @@ -6,9 +6,11 @@ public primitive surface. ## Default -`default = ["std"]`, which implies `alloc`. With `default-features = false`, you get a strict `no_std` build and must opt in to leaf features explicitly. +The default feature set is `["std"]`; `std` implies `alloc`. Set +`default-features = false` for a `no_std` build, then enable every required +algorithm feature explicitly. -## Quick Picks +## Quick picks ```toml # One algorithm, no_std. @@ -36,9 +38,9 @@ rscrypto = { version = "0.7.8", features = ["full", "parallel", "getrandom"] } rscrypto = { version = "0.7.8", features = ["full", "portable-only"] } ``` -## Complete Feature Index +## Complete feature index -### Core Features +### Core features | Feature | Pulls in | Use | |---|---|---| @@ -46,7 +48,7 @@ rscrypto = { version = "0.7.8", features = ["full", "portable-only"] } | `std` | `alloc` | Runtime CPU detection and `std::io` adapters. | | `alloc` | -- | Allocating APIs such as PHC string encoding and `Vec`-returning digest, MAC, AEAD, and signature helpers. | -### Umbrella Features +### Umbrella features | Feature | Pulls in | |---|---| @@ -63,7 +65,7 @@ rscrypto = { version = "0.7.8", features = ["full", "portable-only"] } | `key-exchange` | `x25519`, `ml-kem` | | `aead` | `aes-gcm`, `aes-gcm-siv`, `chacha20poly1305`, `xchacha20poly1305`, `aegis256`, `ascon-aead` | -### Algorithm Leaf Features +### Algorithm leaf features | Feature | Pulls in | Enables | |---|---|---| @@ -102,17 +104,32 @@ rscrypto = { version = "0.7.8", features = ["full", "portable-only"] } | `aegis256` | -- | AEGIS-256 | | `ascon-aead` | -- | Ascon-AEAD128 | -### Auxiliary Features +### Auxiliary features | Feature | Effect | |---|---| -| `getrandom` | Enables fallible OS-RNG constructors such as `try_random()` / `try_generate()`, `RapidRandomState::try_new()`, canonical Argon2id/scrypt password-record generation, ML-KEM `try_generate_keypair()` / `try_encapsulate()`, AEAD random sealing helpers, RSA key generation, signing salt/blinding, encryption randomness, and private-operation blinding. Password-record salts are intentionally OS-owned; other APIs retain caller-supplied byte-filling closures where deterministic tests or constrained integrations need them. Deterministic ECDSA signing does not use OS randomness. RSA key generation uses OS entropy to seed its key-generation HMAC_DRBG; no separate DRBG feature is required. | +| `getrandom` | Adds OS-backed random generation; see below. | | `serde` | Serde for non-secret byte wrappers (nonces, tags, public keys, signatures). | | `serde-secrets` | Serde for secret-key and shared-secret bytes. Implies `serde`. Use only for controlled key-material storage, not logs or DTOs. | | `parallel` | Rayon-backed BLAKE3 and Argon2 lane parallelism. Requires `std`, `blake3`, `argon2`. | | `diag` | Diagnostic introspection of dispatch decisions and selected benchmark-only component hooks. Requires `std`; hidden diagnostic symbols are not stable application API. | | `portable-only` | Makes runtime capability detection report no SIMD/ASM capabilities. See below. | +## `getrandom` + +`getrandom` enables fallible OS-backed constructors such as `try_random()` and +`try_generate()`. It also enables `RapidRandomState::try_new()`, canonical +Argon2id and scrypt password-record generation, ML-KEM +`try_generate_keypair()` and `try_encapsulate()`, AEAD random sealing, and RSA +key generation, signing salt and blinding, encryption randomness, and +private-operation blinding. + +Password-record salts are always OS-generated. Other APIs retain +caller-supplied byte-filling closures for deterministic tests and constrained +integrations. Deterministic ECDSA signing does not use OS randomness. RSA key +generation uses OS entropy to seed its HMAC_DRBG; no separate DRBG feature is +required. + ## `portable-only` `portable-only` makes `platform::caps()` return the empty capability set. @@ -120,6 +137,10 @@ Dispatchers that consult runtime capabilities therefore fall through to portable backends instead of invoking host SIMD/ASM kernels. Use it when a deployment requires runtime dispatch to ignore host acceleration. -This flag does **not** change `platform::caps_static()`, remove SIMD code from the binary, or create a constant-time proof by itself. For binary-level exclusion, also restrict `target-feature` via `RUSTFLAGS`. For release evidence boundaries, use [`constant-time.md`](constant-time.md). +This flag does **not** change `platform::caps_static()`, override a backend +selected at compile time, remove accelerated code from the binary, or create a +constant-time proof. Restrict `target-feature` through `RUSTFLAGS` when the +binary must exclude compile-time accelerated paths. Use +[`constant-time.md`](constant-time.md) for release evidence boundaries. -See [`compliance.md`](compliance.md) for framework-by-framework deployment posture. +See [`compliance.md`](compliance.md) for the FIPS-oriented deployment boundary. diff --git a/docs/migration/README.md b/docs/migration/README.md index 6bd77533..01f2ecb3 100644 --- a/docs/migration/README.md +++ b/docs/migration/README.md @@ -1,14 +1,16 @@ # Migration Guides -This index covers 36 migration guides for API revisions, individual crates, -and larger stacks. +This index covers 36 migration guides. Start with the crate or API you already +use. Each guide identifies dependency, import, call-site, semantic, error, and +unsupported-surface changes. -Each guide identifies dependency, import, call-site, behavior, and unsupported -surface changes. +“Verified against” means the repository tests the mapped operation against the +named version or an independent oracle. It does not mean that rscrypto replaces +the upstream crate's complete API. -If you are evaluating `rscrypto`, start with the crate you already use. -The guide states whether the mapped surface is compatible and which upstream -APIs must remain. +Code blocks are migration fragments unless a guide says otherwise. Names such +as `data`, `key`, `password`, and `plaintext` stand for values already present +at the call site. For projects upgrading rscrypto itself, start with [`misuse-resistant API boundaries`](api-boundaries.md). @@ -36,7 +38,7 @@ For projects upgrading rscrypto itself, start with | [`twox-hash`](twox-hash.md) | `Xxh3`, `Xxh3_128`, `Xxh3Hasher`, `Xxh3_128Hasher`, `Xxh3BuildHasher` | API migration guidance; XXH3 output covered by `xxhash-rust` oracle tests | | [`rapidhash`](rapidhash.md) | `RapidHash64`, `RapidStreamHasher`, `RapidHasher`, `RapidSeededState`, `RapidRandomState` | Verified against `rapidhash 4.5.1` | -## Auth (MAC + KDF) +## Authentication, MACs, and KDFs | From | To | Status | |---|---|---| @@ -56,7 +58,7 @@ For projects upgrading rscrypto itself, start with | [`ascon-aead`](RustCrypto/ascon-aead.md) (RustCrypto) | `AsconAead128` | Verified against `ascon-aead 0.6.0` | | [`aegis`](aegis.md) | `Aegis256` | Verified against `aegis 0.9.12` | -## Signatures + Key Exchange +## Signatures and key exchange | From | To | Status | |---|---|---| @@ -65,14 +67,14 @@ For projects upgrading rscrypto itself, start with | [`rsa`](RustCrypto/rsa.md) (RustCrypto) | `RsaPublicKey`, `RsaPrivateKey`, RSA-PSS, RSASSA-PKCS1-v1_5, OAEP | Partial; verified through CAVP, Wycheproof, and RustCrypto/ring/OpenSSL oracles | | [`x25519-dalek`](RustCrypto/x25519-dalek.md) | `X25519SecretKey`, `X25519PublicKey`, `X25519SharedSecret` | Verified against `x25519-dalek 2.0.1` | -## Password Hashing +## Password hashing | From | To | Status | |---|---|---| | [`argon2`](RustCrypto/argon2.md) (RustCrypto) | Raw `Argon2{d,i,id}` KDFs; bounded `Argon2idPassword` records | Verified against `argon2 0.6.0-rc.8` | | [`scrypt`](RustCrypto/scrypt.md) (RustCrypto) | Raw `Scrypt` KDF; bounded `ScryptPassword` records | Verified against `scrypt 0.12.0` | -## Stack Migrations +## Stack migrations | From | To | Status | |---|---|---| diff --git a/docs/migration/RustCrypto/aes-gcm-siv.md b/docs/migration/RustCrypto/aes-gcm-siv.md index 5db5b6c7..b1d666aa 100644 --- a/docs/migration/RustCrypto/aes-gcm-siv.md +++ b/docs/migration/RustCrypto/aes-gcm-siv.md @@ -1,6 +1,8 @@ # Migration: `aes-gcm-siv` (RustCrypto) → `rscrypto` -> Same algorithm (RFC 8452), same nonce-misuse-resistant guarantees. Replace `Aes256GcmSiv` / `Key` / `Nonce` / `Payload { msg, aad }` with rscrypto's named types and a buffer-style API. +> Replace `Aes256GcmSiv` / `Key` / `Nonce` / +> `Payload { msg, aad }` with rscrypto's named types and a caller-buffer API. +> The RFC 8452 construction and combined ciphertext-and-tag bytes are unchanged. Verified against `aes-gcm-siv = "0.11.1"` and the `rscrypto` 0.7.8 line. Evidence: `tests/aes128gcmsiv_oracle.rs`, `tests/aes256gcmsiv_oracle.rs`, and `tests/aead_wycheproof.rs`. @@ -64,7 +66,7 @@ use rscrypto::{ let key = Aes256GcmSivKey::from_bytes([0u8; 32]); let cipher = Aes256GcmSiv::new(&key); let nonce = Nonce96::from_bytes([0u8; 12]); -let mut ct = vec![0u8; plaintext.len() + 16]; +let mut ct = vec![0u8; Aes256GcmSiv::ciphertext_len(plaintext.len())?]; cipher.encrypt(&nonce, aad, plaintext, &mut ct)?; ``` @@ -76,7 +78,7 @@ define nonce derivation. ```rust // After -let mut plaintext = vec![0u8; ct.len() - 16]; +let mut plaintext = vec![0u8; Aes256GcmSiv::plaintext_len(ct.len())?]; cipher.decrypt(&nonce, aad, &ct, &mut plaintext)?; ``` diff --git a/docs/migration/RustCrypto/aes-gcm.md b/docs/migration/RustCrypto/aes-gcm.md index bf77f172..f0698bae 100644 --- a/docs/migration/RustCrypto/aes-gcm.md +++ b/docs/migration/RustCrypto/aes-gcm.md @@ -67,12 +67,15 @@ use rscrypto::{ let key = Aes256GcmKey::from_bytes([0u8; 32]); let cipher = Aes256Gcm::new(&key); let nonce = Nonce96::from_bytes([0u8; 12]); -let mut ct = vec![0u8; plaintext.len() + 16]; +let mut ct = vec![0u8; Aes256Gcm::ciphertext_len(plaintext.len())?]; cipher.encrypt(&nonce, aad, plaintext, &mut ct)?; // ct[..plaintext.len()] is ciphertext, ct[plaintext.len()..] is the 16-byte tag. ``` The output layout is identical (`[ciphertext || tag]`), so on-the-wire compatibility is preserved. The shape change is who owns the buffer: `aes-gcm` allocates a `Vec`, rscrypto writes into a buffer you pre-sized. +Use `Aes256Gcm::ciphertext_len` and `Aes256Gcm::plaintext_len` before +allocation; they reject length overflow and combined inputs shorter than the +tag. The expert trait import is required because this migration preserves the upstream caller-supplied nonce. New protocols should use `seal_random` or `NonceCounter` so nonce issuance is not a normal call-site choice. @@ -88,7 +91,7 @@ let plaintext = cipher ```rust // After -let mut plaintext = vec![0u8; ct.len() - 16]; +let mut plaintext = vec![0u8; Aes256Gcm::plaintext_len(ct.len())?]; cipher.decrypt(&nonce, aad, &ct, &mut plaintext)?; ``` @@ -154,5 +157,13 @@ cipher.decrypt_in_place(&nonce, aad, &mut buffer, &tag)?; format. - **`AeadInPlace` trait import not needed.** RustCrypto requires importing `aead::AeadInPlace` separately to call the `_in_place_detached` methods. rscrypto exposes both shapes through the single `Aead` trait. - **`generic-array` is gone.** rscrypto does not return `GenericArray` from any AEAD method. Tags are typed newtypes (`Aes256GcmTag`) wrapping `[u8; 16]`; key/nonce types wrap `[u8; N]` directly. -- **Hardware acceleration.** Both crates dispatch to AES-NI on x86_64 and AES-CE on aarch64. rscrypto adds VAES (AVX-512), s390x CPACF, and a portable bitsliced fallback that avoids secret-indexed tables. That source property is not a universal timing proof; constant-time coverage is limited to the compiler, target, features, and binary in the matching [release evidence](../../constant-time.md). Force the portable kernel via `RSCRYPTO_AES_GCM_FORCE=portable` (std only) or the crate's `portable-only` feature. +- **Hardware acceleration.** Both crates dispatch to AES-NI on x86_64 and + AES-CE on aarch64. rscrypto adds VAES (AVX-512), s390x CPACF, and a portable + bitsliced fallback that avoids secret-indexed tables. That source property is + not a universal timing proof; constant-time coverage is limited to the + compiler, target, features, and binary in the matching + [release evidence](../../constant-time.md). The crate's `portable-only` + feature makes runtime capability detection ignore host acceleration but does + not override a compile-time backend; see + [`docs/features.md`](../../features.md#portable-only). - **`no_std`.** Both crates support `no_std`. rscrypto's combined API requires the caller to provide an output buffer, which fits stack-only embedded use. The `vec!` calls in the examples above are for std convenience; in `no_std` they become fixed-size arrays. diff --git a/docs/migration/RustCrypto/argon2.md b/docs/migration/RustCrypto/argon2.md index e2a8c4b9..4ff39026 100644 --- a/docs/migration/RustCrypto/argon2.md +++ b/docs/migration/RustCrypto/argon2.md @@ -136,7 +136,9 @@ Common canonical RustCrypto Argon2id v1.3 records with 32-byte outputs remain ve ## Operational notes -- `Argon2Params::default()` is `m=19_456 KiB, t=2, p=1`, matching the [OWASP Password Storage Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html) baseline as checked on 2026-07-25. +- `Argon2Params::default()` is `m=19_456 KiB, t=2, p=1`, matching the + [OWASP Password Storage Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html) + baseline as checked on 2026-07-29. - Argon2d and Argon2id use data-dependent memory access and are not local side-channel constant-time claims. Argon2i is the data-independent raw variant. - The memory matrix is zeroized on drop. Target-size overflow and allocation failure are distinct errors on raw derivation. - The `parallel` feature enables Rayon lane parallelism when the profile and workload justify it. diff --git a/docs/migration/RustCrypto/ascon-aead.md b/docs/migration/RustCrypto/ascon-aead.md index a407de1d..d7cb6374 100644 --- a/docs/migration/RustCrypto/ascon-aead.md +++ b/docs/migration/RustCrypto/ascon-aead.md @@ -1,6 +1,8 @@ # Migration: `ascon-aead` (RustCrypto) → `rscrypto` -> NIST SP 800-232 lightweight AEAD. Replace `AsconAead128` / `Key` / `Nonce` / `Payload { msg, aad }` with rscrypto's named types and a buffer-style API. 128-bit key, 128-bit nonce, 128-bit tag: all the bytes are 16. +> Replace `AsconAead128` / `Key` / `Nonce` / +> `Payload { msg, aad }` with rscrypto's named types and a caller-buffer API. +> NIST SP 800-232 specifies a 16-byte key, nonce, and tag. Verified against `ascon-aead = "0.6.0"` and the `rscrypto` 0.7.8 line. Evidence: `tests/ascon_aead_oracle.rs`. @@ -60,7 +62,7 @@ use rscrypto::{ let key = AsconAead128Key::from_bytes([0u8; 16]); let cipher = AsconAead128::new(&key); let nonce = Nonce128::from_bytes([0u8; 16]); -let mut ct = vec![0u8; plaintext.len() + 16]; +let mut ct = vec![0u8; AsconAead128::ciphertext_len(plaintext.len())?]; cipher.encrypt(&nonce, aad, plaintext, &mut ct)?; ``` @@ -71,7 +73,7 @@ The expert trait import preserves an existing caller-nonce protocol. Prefer ```rust // After -let mut plaintext = vec![0u8; ct.len() - 16]; +let mut plaintext = vec![0u8; AsconAead128::plaintext_len(ct.len())?]; cipher.decrypt(&nonce, aad, &ct, &mut plaintext)?; ``` @@ -86,13 +88,16 @@ cipher.decrypt_in_place(&nonce, aad, &mut buffer, &tag)?; ## Notes -- **NIST SP 800-232 finalised on 2025-08-13.** Both crates ship the final spec (the 16-byte key / 16-byte nonce / 16-byte tag layout). Outputs are bit-identical (verified in the harness). +- **NIST SP 800-232.** NIST published the final standard on 2025-08-13. + The harness verifies byte-identical output for the final Ascon-AEAD128 + parameter set. - **Implementation boundary.** rscrypto currently uses a portable, table-free implementation. That source structure is not a machine-code timing proof; generated-code constant-time claims are limited to the compiler, target, features, and binary in the matching [release evidence](../../constant-time.md). -- **128-bit key is the only key length.** Ascon-AEAD does not have a 256-bit variant; the 128-bit spec is what NIST standardised. +- **128-bit key is the only key length.** Ascon-AEAD does not have a 256-bit + variant; SP 800-232 specifies the 128-bit parameter set. - **Nonce reuse semantics.** Ascon-AEAD-128 is *not* nonce-misuse-resistant. Reusing `(key, nonce)` reveals plaintext XORs. Prefer deterministic uniqueness. A uniformly random 128-bit nonce has lower collision probability than a uniformly random 96-bit nonce at the same message count, but the deployment must still define a message limit. - **No `Payload`, no `KeyInit` import.** Same simplification as the rest of the AEAD lane. - **Failed-open buffer semantics change.** RustCrypto keeps the in-place buffer diff --git a/docs/migration/RustCrypto/ascon-hash.md b/docs/migration/RustCrypto/ascon-hash.md index 92d948d9..51cada0a 100644 --- a/docs/migration/RustCrypto/ascon-hash.md +++ b/docs/migration/RustCrypto/ascon-hash.md @@ -1,6 +1,8 @@ # Migration: `ascon-hash` (RustCrypto) → `rscrypto` -> Same algorithm (NIST LWC Ascon-Hash256). `ascon_hash::AsconHash256` becomes `rscrypto::AsconHash256`; everything else (trait shape, `update`, `finalize`) carries over. +> Replace `ascon_hash::AsconHash256` with `rscrypto::AsconHash256`. Both map to +> the SP 800-232 Ascon-Hash256 parameter set; the output bytes, `update`, and +> `finalize` flow are unchanged. Verified against `ascon-hash = "0.4.0"` and the `rscrypto` 0.7.8 line. Evidence: `tests/ascon_official_vectors.rs`, `tests/ascon_hash_oracle.rs`, `tests/ascon_cxof_vectors.rs`, and `tests/ascon_differential.rs`. @@ -86,18 +88,23 @@ reader.squeeze(&mut out); ```rust // After (rscrypto) use rscrypto::{AsconCxof128, Xof}; -let mut reader = AsconCxof128::xof(b"customization-string", b"data"); +let mut reader = AsconCxof128::xof(b"customization-string", b"data")?; let mut out = [0u8; 64]; reader.squeeze(&mut out); ``` -The `customization-string` is bounded at 256 bytes by the spec: passing longer returns an `AsconCxofCustomizationError` (use `try_xof` if you need fallible construction). +The spec limits the customization string to 256 bytes. +`AsconCxof128::new`, `AsconCxof128::xof`, and +`AsconCxof128::hash_into` return `AsconCxofCustomizationError` when that limit +is exceeded. ## Notes - **`Output` → `[u8; N]`.** Same as the rest of the RustCrypto migrations: drop `.as_slice()` / `.as_ref()`. - **`finalize` consumes vs. borrows.** Same as the rest: drop any `.clone()`. -- **NIST LWC standard.** Ascon-Hash256 is the lightweight cryptography winner and standardised by NIST in SP 800-232. Both implementations track the final spec. +- **NIST standard.** NIST published Ascon-Hash256, Ascon-XOF128, and + Ascon-CXOF128 in SP 800-232 on 2025-08-13. The differential tests compare + both implementations against the final parameter set. - **Implementation boundary.** rscrypto currently exposes a portable implementation for these Ascon hash and XOF types. `portable-only` does not change their backend selection. diff --git a/docs/migration/RustCrypto/chacha20poly1305.md b/docs/migration/RustCrypto/chacha20poly1305.md index 5e7138da..1fb07d1c 100644 --- a/docs/migration/RustCrypto/chacha20poly1305.md +++ b/docs/migration/RustCrypto/chacha20poly1305.md @@ -1,6 +1,9 @@ # Migration: `chacha20poly1305` (RustCrypto) → `rscrypto` -> Covers both `ChaCha20Poly1305` (96-bit nonce, RFC 8439) and `XChaCha20Poly1305` (192-bit nonce). Same algorithm, byte-identical ciphertext+tag; replace `Key` / `Nonce` / `XNonce` / `Payload { msg, aad }` with `ChaCha20Poly1305Key` + `Nonce96` / `Nonce192` and a buffer-style API. +> Replace `Key` / `Nonce` / `XNonce` / `Payload { msg, aad }` with named +> keys, `Nonce96` or `Nonce192`, and a caller-buffer API. The mapped +> ChaCha20-Poly1305 and XChaCha20-Poly1305 operations preserve ciphertext and +> tag bytes. Verified against `chacha20poly1305 = "0.11.0"` and the `rscrypto` 0.7.8 line. Evidence: `tests/chacha20poly1305.rs`, `tests/xchacha20poly1305.rs`, and `tests/aead_wycheproof.rs`. @@ -62,7 +65,7 @@ use rscrypto::{ let key = ChaCha20Poly1305Key::from_bytes([0u8; 32]); let cipher = ChaCha20Poly1305::new(&key); let nonce = Nonce96::from_bytes([0u8; 12]); -let mut ct = vec![0u8; plaintext.len() + 16]; +let mut ct = vec![0u8; ChaCha20Poly1305::ciphertext_len(plaintext.len())?]; cipher.encrypt(&nonce, aad, plaintext, &mut ct)?; ``` @@ -89,7 +92,7 @@ use rscrypto::{ let key = XChaCha20Poly1305Key::from_bytes([0u8; 32]); let cipher = XChaCha20Poly1305::new(&key); let nonce = Nonce192::from_bytes([0u8; 24]); -let mut ct = vec![0u8; plaintext.len() + 16]; +let mut ct = vec![0u8; XChaCha20Poly1305::ciphertext_len(plaintext.len())?]; cipher.encrypt(&nonce, aad, plaintext, &mut ct)?; ``` @@ -97,15 +100,17 @@ The XChaCha variant uses `Nonce192` (24 bytes). That is the only structural chan The expert trait import is required because these examples preserve an existing caller-nonce protocol. Prefer `seal_random` for new protocols. -### Decrypt + tamper-detection +### ChaCha20-Poly1305 decrypt and tamper detection ```rust // After -let mut plaintext = vec![0u8; ct.len() - 16]; +let mut plaintext = vec![0u8; ChaCha20Poly1305::plaintext_len(ct.len())?]; cipher.decrypt(&nonce, aad, &ct, &mut plaintext)?; // Err(OpenError::Verification(_)) on tag mismatch. ``` +Use `XChaCha20Poly1305::plaintext_len` for the XChaCha variant. + ### Detached (in-place) ```rust @@ -134,5 +139,11 @@ cipher.decrypt_in_place(&nonce, aad, &mut buffer, &tag)?; - **Failed-open buffer semantics change.** RustCrypto keeps the in-place buffer unchanged on error. rscrypto clears it on authentication failure. Combined rscrypto decrypt also clears its output buffer on authentication failure. -- **Software-only acceleration.** ChaCha20 has no hardware AES; both crates use SIMD ChaCha20 implementations. rscrypto runtime-dispatches between SSE2/AVX2/AVX-512 on x86_64 and NEON on aarch64. The always-available portable scalar fallback has fixed-work source structure, but generated-code constant-time coverage is limited to the compiler, target, features, and binary in the matching [release evidence](../../constant-time.md). Force portable via `RSCRYPTO_CHACHA20_POLY1305_FORCE=portable` (std only). +- **Acceleration.** rscrypto selects eligible vector or assembly backends from + detected CPU capabilities and retains a portable scalar fallback. The + fallback has fixed-work source structure, but generated-code constant-time + coverage is limited to the compiler, target, features, and binary in the + matching [release evidence](../../constant-time.md). The `portable-only` + feature constrains runtime dispatch as documented in + [`docs/features.md`](../../features.md#portable-only). - **`no_std`.** Both crates support `no_std`. diff --git a/docs/migration/RustCrypto/ed25519-dalek.md b/docs/migration/RustCrypto/ed25519-dalek.md index 1a49f562..14e28ac7 100644 --- a/docs/migration/RustCrypto/ed25519-dalek.md +++ b/docs/migration/RustCrypto/ed25519-dalek.md @@ -1,6 +1,8 @@ # Migration: `ed25519-dalek` → `rscrypto` -> Replace `SigningKey` / `VerifyingKey` / `Signature` with `Ed25519SecretKey` / `Ed25519PublicKey` / `Ed25519Signature`. Same RFC 8032 algorithm, byte-identical signatures (Ed25519 is deterministic), strict verification on by default. +> Replace `SigningKey` / `VerifyingKey` / `Signature` with +> `Ed25519SecretKey` / `Ed25519PublicKey` / `Ed25519Signature`. Signing +> preserves RFC 8032 output bytes, and the rscrypto verifier is always strict. Verified against `ed25519-dalek = "2.2.0"` and the `rscrypto` 0.7.8 line. Evidence: `tests/ed25519_rfc8032_vectors.rs`, `tests/ed25519_oracle.rs`, and `tests/ed25519_wycheproof.rs`. diff --git a/docs/migration/RustCrypto/hmac.md b/docs/migration/RustCrypto/hmac.md index 627243cd..9f34224a 100644 --- a/docs/migration/RustCrypto/hmac.md +++ b/docs/migration/RustCrypto/hmac.md @@ -106,7 +106,15 @@ let expected_tag = HmacSha256Tag::from_bytes(expected_tag_bytes); HmacSha256::verify_tag(key, data, &expected_tag)?; // Result<(), VerificationError> ``` -Streaming form: `let mut mac = HmacSha256::new(key); mac.update(data); mac.verify(&expected_tag)?;`. Tags are typed (`HmacSha256Tag`, `HmacSha384Tag`, `HmacSha512Tag`, and the `HmacSha3_*Tag` family) and deliberately do not implement `PartialEq` or `Eq`. Prefer `verify` / `verify_tag` for authentication decisions. Direct tag comparison is explicit: `left.ct_eq(&right)` returns an opaque `CtDecision`, and only `.declassify()` exposes a branchable bit. This source-level boundary is not a universal timing proof; constant-time claims remain limited to the compiler, target, features, and binary in the matching [release evidence](../../constant-time.md). Use `as_bytes()` / `to_bytes()` only at protocol serialization boundaries. +For streaming verification, construct `HmacSha256`, call `update`, then call +`verify(&expected_tag)`. HMAC tags are typed and do not implement `PartialEq` +or `Eq`. Prefer `verify` or `verify_tag` for authentication decisions. + +Direct comparison is explicit: `left.ct_eq(&right)` returns an opaque +`CtDecision`, and only `declassify()` exposes a branchable bit. This +source-level boundary is not a machine-code timing proof; claims remain limited +to the matching [release evidence](../../constant-time.md). Use `as_bytes()` or +`to_bytes()` only at protocol serialization boundaries. ## Notes diff --git a/docs/migration/RustCrypto/p256.md b/docs/migration/RustCrypto/p256.md index 9ac833da..affa3d7b 100644 --- a/docs/migration/RustCrypto/p256.md +++ b/docs/migration/RustCrypto/p256.md @@ -1,4 +1,4 @@ -# Migration: RustCrypto `p256` -> `rscrypto` +# Migration: RustCrypto `p256` → `rscrypto` Replace RustCrypto P-256 ECDSA signing and verification call sites with `EcdsaP256SecretKey`, `EcdsaP256PublicKey`, and `EcdsaP256Signature`. diff --git a/docs/migration/RustCrypto/p384.md b/docs/migration/RustCrypto/p384.md index 4193e8af..3032e5c0 100644 --- a/docs/migration/RustCrypto/p384.md +++ b/docs/migration/RustCrypto/p384.md @@ -1,4 +1,4 @@ -# Migration: RustCrypto `p384` -> `rscrypto` +# Migration: RustCrypto `p384` → `rscrypto` Replace RustCrypto P-384 ECDSA signing and verification call sites with `EcdsaP384SecretKey`, `EcdsaP384PublicKey`, and `EcdsaP384Signature`. diff --git a/docs/migration/RustCrypto/pbkdf2.md b/docs/migration/RustCrypto/pbkdf2.md index 625ded0d..e7433ecb 100644 --- a/docs/migration/RustCrypto/pbkdf2.md +++ b/docs/migration/RustCrypto/pbkdf2.md @@ -32,7 +32,7 @@ The `pbkdf2` feature implies `hmac` which implies `sha2`. ## Algorithm map -| `pbkdf2` instantiation | rscrypto type | OWASP Password Storage Cheat Sheet minimum, checked 2026-07-25 | +| `pbkdf2` instantiation | rscrypto type | OWASP Password Storage Cheat Sheet minimum, checked 2026-07-29 | |---|---|---| | `pbkdf2_hmac::` | `Pbkdf2Sha256` | `Pbkdf2Sha256::MIN_RECOMMENDED_ITERATIONS` (600,000) | | `pbkdf2_hmac::` | `Pbkdf2Sha512` | `Pbkdf2Sha512::MIN_RECOMMENDED_ITERATIONS` (220,000) | @@ -104,7 +104,11 @@ Pbkdf2Sha256::verify_password(submitted_password, &stored_salt, stored_iters, &s // Ok(()) on match, Err(VerificationError) on mismatch after full tag comparison ``` -Drop the `subtle` dependency for the verify path. The stateful form is `state.verify(salt, iters, &expected)`, which applies the same default policy. The comparison has content-independent source structure for a public output length, but generated-code constant-time claims remain limited to the compiler, target, features, and binary in the matching [release evidence](../../constant-time.md). +Drop the `subtle` dependency for this verification path. The stateful form, +`state.verify(salt, iters, &expected)`, applies the same default policy. The +comparison traverses the public output length at the source level; generated +code claims remain limited to the matching +[release evidence](../../constant-time.md). ## Notes @@ -114,10 +118,25 @@ Drop the `subtle` dependency for the verify path. The stateful form is `state.ve upstream parser or store separately reviewed algorithm, iteration, salt, and derived-key fields. - **Password helpers reject weak parameters.** `derive_key`, `derive_key_array`, `verify_password`, and stateful `verify` enforce the type-specific minimum iteration count and a 16-byte salt by default. `Pbkdf2Sha256::derive_key_primitive` / `verify_password_primitive` preserve raw PBKDF2 behavior for test vectors and explicit migrations. +- **Password verification bounds work.** Stored-password verification rejects + counts above the variant's `MAX_VERIFY_ITERATIONS` before constructing HMAC + state. Primitive derivation and verification remain unbounded for protocols + that deliberately require a larger count. Existing custom lower-bound + policies remain unbounded for compatibility; use + `verify_with_policy_bounded` or `verify_password_with_policy_bounded` to set + the maximum accepted work for an untrusted record. - **Rejects zero iterations.** The upstream free function has no error channel for rejecting a zero `u32` count. rscrypto's policy and primitive derivation APIs return `Pbkdf2Error::InvalidIterations`; handle that new error path. - **Output length cap (RFC 8018 §5.2 step 1).** PBKDF2 limits output to `(2^32 - 1) * hLen`. `pbkdf2` does not check; rscrypto returns `Err(Pbkdf2Error::OutputTooLong)`. The cap is in the gigabytes: only relevant for adversarial inputs. -- **Policy override.** Use `Pbkdf2VerifyPolicy` and `params_with_policy` only when you have a deliberate migration policy. This keeps legacy acceptance explicit instead of making low-cost password verification the default. -- **Iteration recommendation.** `MIN_RECOMMENDED_ITERATIONS` constants (600,000 for SHA-256, 220,000 for SHA-512) reflect the OWASP Password Storage Cheat Sheet as checked on 2026-07-25. Recheck that external policy before each release that changes these constants. +- **Policy override.** Use `Pbkdf2VerifyPolicy` and `params_with_policy` only + when you have a deliberate migration policy. `allows` and + `params_with_policy` enforce lower bounds; `allows_bounded` and + `params_with_policy_bounded` add the caller-selected upper work limit. This + keeps legacy acceptance explicit without hiding a second policy ceiling. +- **Iteration recommendation.** `MIN_RECOMMENDED_ITERATIONS` constants + (600,000 for SHA-256 and 220,000 for SHA-512) reflect the + [OWASP Password Storage Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html) + as checked on 2026-07-29. Recheck that external policy before changing these + constants. - **`no_std`.** Both crates work in `no_std`. diff --git a/docs/migration/RustCrypto/rsa.md b/docs/migration/RustCrypto/rsa.md index c617cb05..f565ef57 100644 --- a/docs/migration/RustCrypto/rsa.md +++ b/docs/migration/RustCrypto/rsa.md @@ -1,4 +1,4 @@ -# Migration: `rsa` (RustCrypto) -> `rscrypto` +# Migration: `rsa` (RustCrypto) → `rscrypto` rscrypto supports RSA key import/export, key generation, RSA-PSS, RSASSA-PKCS1-v1_5, OAEP, RSAES-PKCS1-v1_5, and protocol profile helpers. This @@ -57,6 +57,11 @@ modern policy: RSA-3072 through RSA-8192 with exponent `65537`. For deployed RSA-2048 compatibility keys, import with `RsaPublicKeyPolicy::legacy_verification()` and the `*_with_policy` parser. +Private-key import additionally requires two conventional half-modulus-width +factors that pass trial division and a 32-base Miller-Rabin probable-prime +screen. Algebraically consistent keys with grossly unbalanced factors or +factors that fail this screen are rejected as `RsaKeyError::InvalidModulus`. + ## Generate Keys ```rust diff --git a/docs/migration/RustCrypto/scrypt.md b/docs/migration/RustCrypto/scrypt.md index 923269c7..5309df45 100644 --- a/docs/migration/RustCrypto/scrypt.md +++ b/docs/migration/RustCrypto/scrypt.md @@ -106,7 +106,9 @@ Common canonical RustCrypto scrypt records with 32-byte outputs remain verifiabl ## Operational notes -- `ScryptParams::default()` is `log_n=17, r=8, p=1`, matching the [OWASP Password Storage Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html) baseline for deployments that cannot use Argon2id, as checked on 2026-07-25. +- `ScryptParams::default()` is `log_n=17, r=8, p=1`, matching the + [OWASP Password Storage Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html) + baseline for deployments that cannot use Argon2id, as checked on 2026-07-29. - The raw algorithm accepts arbitrary salt lengths; generated password records use 16 bytes. - Working buffers are zeroized on drop. Target-size overflow and allocation failure are distinct errors. - scrypt ROMix uses password-derived, data-dependent memory access and is not a local side-channel constant-time claim. The final verifier traverses all expected bytes before returning one opaque result; any generated-code timing claim is limited to the exact configuration in the matching [release evidence](../../constant-time.md). diff --git a/docs/migration/RustCrypto/x25519-dalek.md b/docs/migration/RustCrypto/x25519-dalek.md index ed131326..b2e21aff 100644 --- a/docs/migration/RustCrypto/x25519-dalek.md +++ b/docs/migration/RustCrypto/x25519-dalek.md @@ -1,6 +1,9 @@ # Migration: `x25519-dalek` → `rscrypto` -> Replace `StaticSecret` / `EphemeralSecret` / `PublicKey` / `SharedSecret` with rscrypto's unified `X25519SecretKey` / `X25519PublicKey` / `X25519SharedSecret`. Same RFC 7748 algorithm, byte-identical shared secrets, plus an explicit error on low-order peer input. +> Replace `StaticSecret` / `EphemeralSecret` / `PublicKey` / `SharedSecret` +> with rscrypto's unified `X25519SecretKey` / `X25519PublicKey` / +> `X25519SharedSecret`. Valid RFC 7748 inputs produce the same shared-secret +> bytes; rscrypto returns an explicit error for an all-zero result. Verified against `x25519-dalek = "2.0.1"` and the `rscrypto` 0.7.8 line. Evidence: `tests/x25519_vectors.rs`, `tests/x25519_oracle.rs`, and `tests/x25519_wycheproof.rs`. @@ -84,7 +87,11 @@ let shared = secret.diffie_hellman(&peer_public)?; // Result` overwrite their owned bytes on drop. The claim does not extend to caller-created copies; see [`docs/secret-ownership.md`](../../secret-ownership.md). -- **No HKDF integration.** Some protocols KDF the X25519 shared secret immediately (e.g., Noise, Signal). rscrypto's `HkdfSha256` (from `features = ["hkdf"]`) is the natural pairing; see `RustCrypto/hkdf.md`. +- **No HKDF integration.** Some protocols immediately pass the X25519 shared + secret to a KDF. `HkdfSha256` is available through `features = ["hkdf"]`; + the protocol must define the KDF and transcript binding. See the + [`hkdf` migration guide](hkdf.md). - **Scalar-multiplication timing.** rscrypto's portable backend uses fixed-work field-arithmetic source structure, and accelerated paths are differential-tested against it. That source property does not prove every compiler/target binary; constant-time coverage is limited to the exact configurations in the matching [release evidence](../../constant-time.md). - **`no_std`.** Both crates support `no_std` with no `alloc` requirement. The `getrandom`-backed `try_generate()` requires `getrandom`; the closure form `try_generate_with(|buf| ...)` does not. diff --git a/docs/migration/aegis.md b/docs/migration/aegis.md index edcd36ff..83796170 100644 --- a/docs/migration/aegis.md +++ b/docs/migration/aegis.md @@ -1,6 +1,8 @@ # Migration: `aegis` (jedisct1) → `rscrypto` -> Covers AEGIS-256 from the `aegis` crate. Replace `Aegis256::::new(&key, &nonce).encrypt(msg, aad) -> (Vec, [u8; TAG])` with rscrypto's `Aead`-trait-style `encrypt(&nonce, aad, msg, &mut out)`. Same algorithm (draft-irtf-cfrg-aegis-aead), byte-identical ciphertext+tag. +> Replace `Aegis256::::new(&key, &nonce).encrypt(msg, aad)` with +> rscrypto's caller-buffer `encrypt(&nonce, aad, msg, &mut out)`. The mapped +> AEGIS-256 operation preserves ciphertext and tag bytes. Verified against `aegis = "0.9.12"` and the `rscrypto` 0.7.8 line. Evidence: `tests/aegis256_oracle.rs` and `tests/aead_wycheproof.rs`. @@ -60,7 +62,7 @@ use rscrypto::{ let key = Aegis256Key::from_bytes([0u8; 32]); let cipher = Aegis256::new(&key); // key at construction let nonce = Nonce256::from_bytes([0u8; 32]); // nonce at call -let mut ct = vec![0u8; plaintext.len() + 16]; +let mut ct = vec![0u8; Aegis256::ciphertext_len(plaintext.len())?]; cipher.encrypt(&nonce, aad, plaintext, &mut ct)?; // appended tag ``` @@ -101,7 +103,7 @@ let plaintext = cipher.decrypt(&ciphertext, &tag, aad)?; // returns Result` or `Aegis256<32>`; rscrypto exposes only the 16-byte-tag variant. Keep `aegis` diff --git a/docs/migration/api-boundaries.md b/docs/migration/api-boundaries.md index 162ee7de..5ceb929c 100644 --- a/docs/migration/api-boundaries.md +++ b/docs/migration/api-boundaries.md @@ -1,8 +1,8 @@ # Migration: misuse-resistant API boundaries -This release makes invalid lengths and caller-controlled expert operations -explicit. Cryptographic kernels, output bytes, buffer layouts, and dispatch -behavior are unchanged. +These API changes move invalid lengths and caller-controlled expert operations +to explicit boundaries. They do not change cryptographic outputs, buffer +layouts, or backend dispatch. ## BLAKE2 @@ -74,5 +74,4 @@ nonce uniqueness for the key. - Diagnostic hooks remain under their owning modules such as `auth`, `aead`, and `hashes`; they are no longer re-exported from the crate root. -These are namespace changes only. They add no allocation, dynamic dispatch, -registry, lock, or extra cryptographic work. +These namespace changes do not add a registry, lock, or dynamic dispatch. diff --git a/docs/migration/aws-lc-rs.md b/docs/migration/aws-lc-rs.md index b27e3f48..a8fab39c 100644 --- a/docs/migration/aws-lc-rs.md +++ b/docs/migration/aws-lc-rs.md @@ -1,4 +1,4 @@ -# Migration: `aws-lc-rs` -> `rscrypto` +# Migration: `aws-lc-rs` → `rscrypto` `aws-lc-rs` is a safe wrapper around AWS-LC. Migrate it primitive-by-primitive. Do not treat it as a whole-crate swap. @@ -136,14 +136,15 @@ key.seal_in_place_append_tag( ```rust // After use rscrypto::{ - Aes256Gcm, Aes256GcmKey, + Aead, Aes256Gcm, Aes256GcmKey, aead::{Nonce96, expert::AeadWithNonce}, }; let cipher = Aes256Gcm::new(&Aes256GcmKey::from_bytes(*key_bytes)); let nonce = Nonce96::from_bytes(*nonce_bytes); -let mut ciphertext_and_tag = vec![0u8; plaintext.len() + 16]; +let mut ciphertext_and_tag = + vec![0u8; Aes256Gcm::ciphertext_len(plaintext.len())?]; cipher.encrypt(&nonce, aad, plaintext, &mut ciphertext_and_tag)?; ``` diff --git a/docs/migration/aws-lc-sys.md b/docs/migration/aws-lc-sys.md index 4b19736b..2f49d48c 100644 --- a/docs/migration/aws-lc-sys.md +++ b/docs/migration/aws-lc-sys.md @@ -1,9 +1,9 @@ -# Migration: `aws-lc-sys` -> `rscrypto` +# Migration: `aws-lc-sys` → `rscrypto` There is no direct `aws-lc-sys` migration. `aws-lc-sys` exposes the low-level C FFI surface for AWS-LC; rscrypto exposes safe Rust primitive APIs. -## What To Do Instead +## What to do instead If your code uses `aws-lc-sys` directly, first identify the safe operation you need: diff --git a/docs/migration/crc-fast.md b/docs/migration/crc-fast.md index ca1e0fa8..dc0bf553 100644 --- a/docs/migration/crc-fast.md +++ b/docs/migration/crc-fast.md @@ -1,9 +1,12 @@ # Migration: `crc-fast` → `rscrypto` -> Replace `crc-fast`'s enum-driven `checksum(CrcAlgorithm::*, data)` calls with rscrypto's named CRC types. SIMD coverage is comparable on x86_64 and aarch64; rscrypto adds Power, s390x, and RISC-V kernels and removes the `u64`-everywhere return type. +Replace `crc-fast`'s enum-driven `checksum(CrcAlgorithm::*, data)` calls with +rscrypto's named CRC types. The mapped algorithms keep the same CRC parameters, +but rscrypto returns each algorithm's natural integer width instead of `u64`. Verified against `crc-fast = "1.10.0"` and the `rscrypto` 0.7.8 line. -Evidence: `tests/crc16_properties.rs`, `tests/crc32_properties.rs`, and `tests/crc64_properties.rs`. +Evidence: `tests/crc16_properties.rs`, `tests/crc32_properties.rs`, and +`tests/crc64_properties.rs`. ## TL;DR @@ -27,7 +30,8 @@ crc-fast = "1.10" rscrypto = { version = "0.7.8", features = ["crc32", "crc64"] } ``` -Drop `crc16` / `crc24` from the feature list if you don't use them. `features = ["checksums"]` is the umbrella shortcut. +Add `crc16` or `crc24` only when you migrate a mapped variant from those +families. Use `features = ["checksums"]` to enable every CRC family. ## Algorithm map @@ -83,7 +87,9 @@ hasher.update(b"bar"); let value = hasher.finalize(); // borrows &self, returns u32 ``` -`Digest::finalize` consumes `self` in `crc-fast`; `Checksum::finalize` borrows in rscrypto. Drop the rebuild if you were rebuilding a `Digest` per chunk only to keep ownership. +`Digest::finalize` consumes `self` in `crc-fast`; `Checksum::finalize` borrows +in rscrypto. You can read an intermediate finalized value and continue updating +the same rscrypto hasher. ### Combine (parallel chunks) diff --git a/docs/migration/crc32c.md b/docs/migration/crc32c.md index dba6fcbc..6c65db57 100644 --- a/docs/migration/crc32c.md +++ b/docs/migration/crc32c.md @@ -97,7 +97,7 @@ let crc = h.finalize(); - **Single algorithm.** `crc32c` is CRC-32C (Castagnoli) only. If you also need CRC-32 IEEE, both crates would normally require separate dependencies: rscrypto's `crc32` feature covers both with one dep. - **Hardware acceleration parity.** Both crates dispatch to the SSE4.2 `crc32` instruction on x86_64 and the ARMv8 CRC extension on aarch64. rscrypto adds VPCLMULQDQ folding (large buffers on x86_64), SVE2-PMULL (aarch64), VPMSUMD (Power), VGFM (s390x), and Zbc/Zvbc (RISC-V). -- **Force a backend.** `RSCRYPTO_CRC32C_FORCE=portable` selects the portable +- **Force a backend.** `RSCRYPTO_CRC32_FORCE=portable` selects the portable CRC-32C runtime backend in `std` builds. The crate's `portable-only` feature makes runtime capability detection ignore host acceleration; see [`docs/features.md`](../features.md#portable-only) for its limits. diff --git a/docs/migration/dryoc.md b/docs/migration/dryoc.md index 62d2f6bd..a134aad4 100644 --- a/docs/migration/dryoc.md +++ b/docs/migration/dryoc.md @@ -1,4 +1,4 @@ -# Migration: `dryoc` -> `rscrypto` +# Migration: `dryoc` → `rscrypto` `dryoc` follows libsodium's API shape. Migrate the primitive calls directly and keep `dryoc` for libsodium protocol helpers such as `secretbox`, `box`, sealed diff --git a/docs/migration/openssl.md b/docs/migration/openssl.md index a9d06a04..efbe7671 100644 --- a/docs/migration/openssl.md +++ b/docs/migration/openssl.md @@ -1,4 +1,4 @@ -# Migration: `openssl` -> `rscrypto` +# Migration: `openssl` → `rscrypto` `openssl` is a binding to a full system cryptography, TLS, X.509, and provider toolkit. rscrypto can replace selected primitive operations. It does not replace @@ -33,10 +33,15 @@ rscrypto = { version = "0.7.8", default-features = false, features = ["sha2", "h 1. Separate primitive calls from platform calls. 2. Move primitive calls to the matching focused guide: - - hashes: `RustCrypto/sha2.md`, `RustCrypto/sha3.md`, `RustCrypto/blake2.md` - - MAC/KDF: `RustCrypto/hmac.md`, `RustCrypto/hkdf.md`, `RustCrypto/pbkdf2.md` - - AEAD: `RustCrypto/aes-gcm.md`, `RustCrypto/chacha20poly1305.md` - - signatures/RSA: `RustCrypto/ed25519-dalek.md`, `RustCrypto/rsa.md` + - hashes: [`sha2`](RustCrypto/sha2.md), [`sha3`](RustCrypto/sha3.md), and + [`blake2`](RustCrypto/blake2.md); + - MACs and KDFs: [`hmac`](RustCrypto/hmac.md), + [`hkdf`](RustCrypto/hkdf.md), and [`pbkdf2`](RustCrypto/pbkdf2.md); + - AEAD: [`aes-gcm`](RustCrypto/aes-gcm.md) and + [`chacha20poly1305`](RustCrypto/chacha20poly1305.md); + - signatures and RSA: + [`ed25519-dalek`](RustCrypto/ed25519-dalek.md) and + [`rsa`](RustCrypto/rsa.md). 3. Keep OpenSSL for TLS, PKI, provider, engine, and FIPS-provider behavior. ## RSA Boundary @@ -46,6 +51,6 @@ RSAES-PKCS1-v1_5 for the implemented SHA-2 profiles. The RSA test lane checks interoperability against RustCrypto `rsa`, `ring`, AWS-LC, CAVP/Wycheproof vectors, and OpenSSL CLI when it is installed. -Use `RustCrypto/rsa.md` for concrete rscrypto RSA call-site examples. Keep -OpenSSL if you need certificate validation, provider configuration, engine -integration, or OpenSSL FIPS provider semantics. +Use the [`rsa` migration guide](RustCrypto/rsa.md) for concrete rscrypto RSA +call-site examples. Keep OpenSSL if you need certificate validation, provider +configuration, engine integration, or OpenSSL FIPS provider semantics. diff --git a/docs/migration/ring.md b/docs/migration/ring.md index 570e6d33..d6664eec 100644 --- a/docs/migration/ring.md +++ b/docs/migration/ring.md @@ -1,4 +1,4 @@ -# Migration: `ring` -> `rscrypto` +# Migration: `ring` → `rscrypto` `ring` mixes primitive APIs with protocol-shaped helpers. Migrate the primitive surfaces directly; keep `ring` where you need ECDH P-256/P-384, random, or @@ -131,14 +131,15 @@ key.seal_in_place_append_tag( ```rust // After use rscrypto::{ - Aes256Gcm, Aes256GcmKey, + Aead, Aes256Gcm, Aes256GcmKey, aead::{Nonce96, expert::AeadWithNonce}, }; let cipher = Aes256Gcm::new(&Aes256GcmKey::from_bytes(*key_bytes)); let nonce = Nonce96::from_bytes(*nonce_bytes); -let mut ciphertext_and_tag = vec![0u8; plaintext.len() + 16]; +let mut ciphertext_and_tag = + vec![0u8; Aes256Gcm::ciphertext_len(plaintext.len())?]; cipher.encrypt(&nonce, aad, plaintext, &mut ciphertext_and_tag)?; ``` diff --git a/docs/migration/sha3-kmac.md b/docs/migration/sha3-kmac.md index 0d1cb7cd..47ddf764 100644 --- a/docs/migration/sha3-kmac.md +++ b/docs/migration/sha3-kmac.md @@ -129,7 +129,16 @@ use rscrypto::Kmac256; Kmac256::verify_tag(key, custom, data, &expected)?; // Result<(), VerificationError> ``` -Streaming form: `let mut k = Kmac256::new(key, custom); k.update(data); k.verify(&expected)?;`. Drop the `subtle` dependency. Verification traverses the public-length expected tag before returning one opaque result; generated-code constant-time claims remain limited to the exact configuration in the matching [release evidence](../constant-time.md). +For streaming verification, construct `Kmac256`, call `update`, then call +`verify(&expected)`. The authentication helpers require at least 16 bytes for +KMAC128 and 32 bytes for KMAC256, preserving the named security strength. + +Use `verify_primitive` or `verify_tag_primitive` only when a protocol specifies +a shorter output and defines its forgery budget and failed-attempt limit. +Arbitrary-length `finalize_into` and `mac_into` remain available for PRF or KDF +use. Verification traverses the public-length expected tag before returning one +opaque result. Generated-code timing claims remain limited to the matching +[release evidence](../constant-time.md). ## Notes diff --git a/docs/migration/tiny-keccak.md b/docs/migration/tiny-keccak.md index c1b273e1..e22ab527 100644 --- a/docs/migration/tiny-keccak.md +++ b/docs/migration/tiny-keccak.md @@ -131,7 +131,17 @@ use rscrypto::Kmac256; Kmac256::verify_tag(key, custom, data, &expected)?; // Result<(), VerificationError> ``` -Drop the `subtle` dependency for the verify path. Streaming form: `let mut k = Kmac256::new(key, custom); k.update(data); k.verify(&expected)?;`. Verification traverses the public-length expected tag before returning one opaque result; generated-code constant-time claims remain limited to the exact configuration in the matching [release evidence](../constant-time.md). +Drop the `subtle` dependency for this verification path. For streaming +verification, construct `Kmac256`, call `update`, then call +`verify(&expected)`. The authentication helpers require at least 16 bytes for +KMAC128 and 32 bytes for KMAC256, preserving the named security strength. + +Use `verify_primitive` or `verify_tag_primitive` only when a protocol specifies +a shorter output and defines its forgery budget and failed-attempt limit. +Arbitrary-length `finalize_into` and `mac_into` remain available for PRF or KDF +use. Verification traverses the public-length expected tag before returning one +opaque result. Generated-code timing claims remain limited to the matching +[release evidence](../constant-time.md). ### cSHAKE256: XOF streaming diff --git a/docs/platforms.md b/docs/platforms.md index e7a85bbc..d672ad9a 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -1,10 +1,10 @@ # Platforms -`rscrypto` uses a three-tier dispatch model. The same source builds with a -portable Rust path on every supported target and uses hardware backends only -when the target and CPU support them. +Every supported target retains a portable Rust implementation. Compile-time +configuration and, with `std`, runtime CPU detection may select an eligible +accelerated backend. -## Dispatch Model +## Dispatch model 1. **Compile-time**: `#[cfg(target_feature = "...")]` selects the strongest backend permitted by `RUSTFLAGS` / `target-feature`. 2. **Runtime detection** (`std` only): cached `platform::caps()` probes CPU features once via detection intrinsics and, on supported Linux/Android targets, OS capability files such as `/proc/self/auxv`; it then dispatches to the strongest available kernel. @@ -18,7 +18,7 @@ through to portable backends. It does not remove SIMD code from the binary or override compile-time `target_feature` selection. See [`features.md`](features.md#portable-only). -## Acceleration Matrix +## Acceleration matrix Backend availability depends on what the target CPU advertises and what `target-feature` permits. The portable Rust fallback is present on every target @@ -34,7 +34,7 @@ lower tier, including portable Rust. | x86_64 | SSE4.2 CRC32; SSSE3 / PCLMULQDQ; AVX2; AES-NI; SHA-NI; AVX-512F / VL / BW / DQ; AVX-512IFMA; VPCLMULQDQ; VAES | | aarch64 / Apple Silicon | NEON; AES; PMULL; CRC; SHA2; SHA3 / EOR3; SHA512; SVE2-PMULL where available | | s390x (IBM Z) | z/Vector; vector enhancements; CPACF / MSA; VGFM; fixed-work ML-KEM arithmetic | -| ppc64le (POWER) | AltiVec; VSX; POWER8 vector / crypto; POWER9 / POWER10 vector; VPMSUMD | +| ppc64le (POWER) | AltiVec; VSX; POWER8 vector / crypto and atomics; POWER9 / POWER10 vector; VPMSUMD | | riscv64 | V / RVV; Zbc; Zvbc; Zbkc; Zkne / Zknd; Zvkned; Zkt / Zvkt | | wasm32 | SIMD128 where enabled | @@ -48,7 +48,7 @@ ML-KEM arithmetic uses fixed-work z/Vector kernels where those kernels are compiled and selected. The implementation does not replace constant-time hardening with native scalar multiply or divide on secret-fed arithmetic. -## `no_std` Targets +## `no_std` targets The following `no_std` targets are built in CI: @@ -61,9 +61,13 @@ The following `no_std` targets are built in CI: Targets outside this list are not part of the CI contract. Open an issue with the exact target triple and feature set when a required target is missing. +Build coverage does not establish a constant-time claim; use +[`constant-time.md`](constant-time.md) for release-evidenced configurations. -## Per-Platform Benchmark Scorecard +## Per-platform benchmark evidence -Current geomean speedups by platform live in +The historical 2026-07-04 per-platform results live in [`benchmark_results/OVERVIEW.md`](../benchmark_results/OVERVIEW.md#coverage-matrix). -The current public set is the 2026-07-04 nine-runner Linux CI matrix. +Its aggregate includes the equivalent-work limitation documented in +[`benchmarking.md`](benchmarking.md). Benchmark the deployment workload on its +target CPU before choosing a performance-sensitive backend or feature set. diff --git a/docs/release.md b/docs/release.md index cc62acce..aaea3002 100644 --- a/docs/release.md +++ b/docs/release.md @@ -157,7 +157,7 @@ This is the only routine local check that reads live GitHub settings. It writes the captured JSON to `target/repository-controls.json`; normal checks and pre-push validation remain offline. -## What the tag workflow proves +## What the tag workflow verifies Pushing a `vX.Y.Z` tag starts the `Release` workflow. Before crates.io can receive anything, the workflow: @@ -183,10 +183,12 @@ constant-time claim. ## Recovery -Re-running the workflow is safe after a partial failure. If crates.io already has the version, the workflow downloads -it and compares SHA-256 before touching the GitHub Release. A draft release can be repaired and then published. A -published immutable release is never overwritten; the workflow verifies its release attestation and stable crate and -source assets before publishing to crates.io. A mismatch is a hard stop. +After a partial failure, rerun the workflow on the same tag and commit. If +crates.io already contains the version, the workflow downloads it and compares +its SHA-256 before touching the GitHub Release. The workflow can repair and +publish a draft release. It never overwrites a published immutable release; it +verifies the release attestation and stable crate and source assets before +publishing to crates.io. Any mismatch stops the release. If the signed-tag key changes, update `.github/allowed-signers` in a reviewed commit before creating the next release tag. @@ -213,13 +215,16 @@ gh attestation verify rscrypto-X.Y.Z-ct-evidence.tar.gz --repo loadingalias/rscr gh attestation verify rscrypto-X.Y.Z-repository-controls.json --repo loadingalias/rscrypto gh attestation verify rscrypto-X.Y.Z-release-manifest.json --repo loadingalias/rscrypto gh attestation verify SHA256SUMS --repo loadingalias/rscrypto -mkdir ct-evidence && tar -xzf rscrypto-X.Y.Z-ct-evidence.tar.gz -C ct-evidence -(cd ct-evidence && sha256sum --check CT-EVIDENCE-MANIFEST.txt) +ct_evidence_dir=$(mktemp -d) +tar -xzf rscrypto-X.Y.Z-ct-evidence.tar.gz -C "$ct_evidence_dir" +(cd "$ct_evidence_dir" && sha256sum --check CT-EVIDENCE-MANIFEST.txt) ``` -The crate downloaded from crates.io should have the same SHA-256 as the attested release artifact. The release -identity manifest is the machine-readable join between the release's source, artifacts, evidence, and toolchain. The -repository controls JSON records the expected policies, immutable-release setting, live branch and tag rulesets, -effective rules on the default branch, capture time, and release commit. Its validation fields state whether each -bypass list and the immutable-release setting were visible to the capturing token. The JSON is evidence of the -release-time configuration, not a claim that GitHub settings cannot change later. +The crate downloaded from crates.io must have the same SHA-256 as the attested +release artifact. The release identity manifest joins the release source, +artifacts, evidence, and toolchain. The repository-controls JSON records the +expected policies, immutable-release setting, live branch and tag rulesets, +effective default-branch rules, capture time, and release commit. Its +validation fields state whether the capturing token could inspect each bypass +list and the immutable-release setting. The JSON records release-time +configuration; GitHub settings can change afterward. diff --git a/docs/secret-lifecycle.md b/docs/secret-lifecycle.md index 0bfc23e4..055dbf82 100644 --- a/docs/secret-lifecycle.md +++ b/docs/secret-lifecycle.md @@ -1,4 +1,4 @@ -# Secret Lifecycle And Diagnostic Redaction +# Secret lifecycle and diagnostic redaction This document records where rscrypto-owned secret storage is cleared and how that source-level policy is checked after release optimization. The type and @@ -6,7 +6,7 @@ capability boundary is defined in [`secret-ownership.md`](secret-ownership.md); this document follows those owners through success, failure, early return, transfer, reuse, and drop. -## Claim Boundary +## Claim boundary The lifecycle claim covers named owner storage and explicit source-level temporaries: inline arrays and words, initialized `MaybeUninit` regions, @@ -20,7 +20,7 @@ pages, crash dumps, or hardware-backed storage. Protocol-visible tags, keyed outputs, signatures, public keys, nonces, and ciphertexts are not confidential owners merely because they are produced by secret-bearing operations. -## Source Ownership And Cleanup +## Source ownership and cleanup | Owner or flow | Retained secret state | Cleanup boundary | |---|---|---| @@ -40,22 +40,22 @@ or ABI spill slots by the compiler. Their durable source owners and explicit stack/heap temporaries are covered above; the compiler-created copies remain an explicit limitation rather than an unprovable erasure claim. -## Path Audit +## Path audit | Path | Source audit result | Optimized evidence | |---|---|---| -| Success | Finalized HMAC/Keccak/BLAKE copies, oversized-key digests, emitted BLAKE3 blocks, parser staging, and private-operation scratch are cleared after the last read | Fixed stack, secret hex success, HMAC-SHA-2/SHA-3 finalization, and keyed BLAKE3 wrappers retain volatile zero stores | -| Error | `ZeroizingBytes` and RAII owners cover parser/generator failure; AEAD and RSA clear rejected plaintext/private output | The secret hex error wrapper reaches the same audited parser as success, and its release IR contains both cleanup paths | -| Early return | Scope-owned fixed and heap secrets retain `Drop` cleanup across `return` and `?`; explicit cleanup precedes returns from manual scratch paths | `diag_zeroize_early_return` retains zero stores in release MIR, LLVM IR, and assembly | +| Success | Finalized HMAC/Keccak/BLAKE copies, oversized-key digests, emitted BLAKE3 blocks, AEAD authentication state, parser staging, and private-operation scratch are cleared after the last read | Fixed stack, secret hex success, HMAC-SHA-2/SHA-3 finalization, keyed BLAKE3, and portable AEAD authentication wrappers retain volatile zero stores | +| Error | `ZeroizingBytes` and RAII owners cover parser/generator failure; AEAD and RSA clear rejected plaintext/private output | The secret hex error wrapper reaches the same audited parser as success; RSA private-component validation routes every initialized secret buffer through the audited owner drop before deallocation | +| Early return | Scope-owned fixed and heap secrets retain `Drop` cleanup across `return` and `?`; explicit cleanup precedes returns from manual scratch paths | `diag_zeroize_early_return` retains zero stores; RSA validation stages retain the nested owner-drop chain in release MIR, LLVM IR, and assembly | | Move or transfer | `SecretBytes::expose` clears its source before returning ordinary bytes; `SecretVec::into_unprotected_vec` transfers the allocation and responsibility; keyed XOF moves transfer one root owner whose destination clears on drop | Fixed-owner move and keyed BLAKE3 XOF move/consume wrappers retain source and destination cleanup | | Reuse | HMAC clears replaced live SHA state; secret-mode Keccak assignment drops the replaced state; BLAKE3 replacement drops the old owner; BLAKE3 parallel vectors are wiped before reuse | The BLAKE3 reset wrapper contains separate production `Drop` calls for the replaced and final owners | | Drop | Every confidential public owner in the ownership inventory reaches a concrete or nested cleanup implementation; heap owners traverse initialized storage before deallocation | The gate follows the keyed BLAKE3 wrapper into its production `drop_in_place` and requires retained owner and heap-scratch zero stores | -## Release-Binary Inspection +## Inspect an optimized binary Run: -```text +```sh just check-zeroize-evidence ``` @@ -73,15 +73,21 @@ The gate maps evidence to production behavior as follows: | `diag_zeroize_fixed_move`, `diag_zeroize_early_return` | Ownership transfer and early return | | `diag_zeroize_hex_success`, `diag_zeroize_hex_error` | Shared secret-parser success and error cleanup | | `diag_zeroize_hmac_sha256_finalize`, `diag_zeroize_hmac_sha3_finalize` | SHA-2 and Keccak keyed finalization, temporary cleanup, and owner drop | +| `diag_hkdf_sha256_derive_portable`, `diag_hkdf_sha384_derive_portable`, `diag_hkdf_sha512_derive_portable` | HKDF SHA-256, SHA-384, and SHA-512 prefix-owner and expansion-scratch cleanup | +| `diag_zeroize_ecdsa_p256_platform_scratch`, `diag_zeroize_ecdsa_p384_platform_scratch` | Accelerated ECDSA wide-input reduction and modular-inversion workspace cleanup | | `diag_zeroize_blake3_drop`, `diag_zeroize_blake3_reuse` | Production keyed owner drop and replaced-state cleanup | | `diag_zeroize_blake3_xof_move`, `diag_zeroize_blake3_xof_consume` | Keyed XOF ownership transfer and destination drop | | `diag_zeroize_blake3_thread_scratch`, `diag_zeroize_blake3_parallel_scratch` | Thread-local and per-state heap CV wipe before reuse or deallocation | +| `diag_poly1305_block_portable_digest`, `diag_ascon_aead128_tag_portable`, `diag_aegis256_update_portable` | Portable Poly1305, Ascon-AEAD, and AEGIS-256 authentication-state cleanup | +| `diag_aes128gcm_ghash`, `diag_aes256gcm_ghash` | AES-GCM authentication-accumulator cleanup | +| `diag_zeroize_mlkem_sha3_512`, `diag_zeroize_mlkem_shake256_{scalar,pair,quad}` | ML-KEM secret SHA3-512 and scalar, paired, or quad SHAKE256 owner and seeded-state cleanup | +| `diag_rsa_validate_pkcs8_private_key_der_stage` | RSA private-component validation success, staged exits, and errors through heap-owner drop before deallocation | This is host-binary evidence, not a universal machine-code proof. The gate must run on each target whose generated cleanup is being claimed; unsupported architectures retain the source audit only. -## Formatting And Error Audit +## Formatting and error audit [`tests/secret_redaction.rs`](../tests/secret_redaction.rs) holds exact `Debug` and error snapshots for generic secret wrappers, AEAD keys and contexts, @@ -99,8 +105,8 @@ standard error-source chain; callers can still recover it by explicitly matching the public `Random` variant. Other reviewed errors contain only discriminants, public sizes, or opaque verification failures. -`expert::DisplaySecret` is the deliberate exception: constructing it explicitly opts -into rendering borrowed secret bytes. Feature-gated diagnostic functions may -return their declared result bytes, but they do not implicitly format owning -keys, seeds, nonce material, intermediate state, or unmasked shares through -`Debug` or an error value. +`expert::DisplaySecret` is the deliberate exception: constructing it +explicitly opts into rendering borrowed secret bytes. Feature-gated diagnostic +functions may return their declared result bytes, but they do not implicitly +format owning keys, seeds, nonce material, intermediate state, or unmasked +shares through `Debug` or an error value. diff --git a/docs/secret-ownership.md b/docs/secret-ownership.md index c73853b2..54eaf624 100644 --- a/docs/secret-ownership.md +++ b/docs/secret-ownership.md @@ -1,4 +1,4 @@ -# Secret Ownership And Generic Capabilities +# Secret ownership and generic capabilities This inventory identifies named types that retain secret material or secret-derived state, then records where generic capabilities can duplicate, @@ -18,8 +18,8 @@ ciphertexts, PHC records, password-hash parameters, and public RSA scratch are not semantic secret owners. They can still receive sensitive caller data. Operation-local arrays, scalars, limbs, plaintext buffers, and individual backend transfer records are grouped here only where their type-level -`Clone`/`Copy` capability affects ownership. Their complete data flow belongs -to the temporary-flow review. +`Clone`/`Copy` capability affects ownership. This inventory does not enumerate +every operation-local temporary. In the tables below, “explicit duplicate” means a method named `duplicate_secret()` rather than `Clone`. “Masked” means `Debug` can identify @@ -27,7 +27,7 @@ the type or show public metadata but does not print the owned secret bytes. The use column explains why a capability exists; it does not pre-approve that capability for permanent retention. -## Confidential Public Owners And Views +## Confidential public owners and views | Type or family | Clone / Copy | Debug | Serialization or export | Storage | Capability use | |---|---|---|---|---|---| @@ -56,11 +56,13 @@ capability for permanent retention. | `Blake3` | `Clone` | Masked | None | Inline, plus `Vec` scratch with `parallel` | Fork a streamed common prefix in keyed or derive-key mode; cloning also duplicates initialized parallel scratch | | `Blake3XofReader` | `Clone` | Masked | None | Inline | Checkpoint or fork an output cursor; the reader is secret-bearing when created from keyed or derive-key state | -## Protocol-Visible Authentication Owners +## Protocol-visible authentication owners -These owners provide constant-work comparison semantics, but their bytes are -normally transmitted or stored with the protected message. Copying, rendering, -and ordinary serialization therefore do not duplicate a confidential key. +These owners use fixed-length, full-traversal comparison at the source level. +Their bytes are normally transmitted or stored with the protected message, so +copying, rendering, and ordinary serialization do not duplicate a confidential +key. Generated-code timing claims remain limited by +[`constant-time.md`](constant-time.md). | Type or family | Clone / Copy | Debug | Serialization | Storage | Capability use | |---|---|---|---|---|---| @@ -69,7 +71,7 @@ and ordinary serialization therefore do not duplicate a confidential key. | `Poly1305Tag` | `Clone + Copy` | Raw hex | None | Inline | Detached one-time authenticator transport and verification | | `Blake3KeyedHash` | `Clone + Copy` | Raw hex | None | Inline | Protocol-visible keyed output with sealed `ct_eq` verification | -## Internal And Operation-Scoped Owners +## Internal and operation-scoped owners | Owner | Clone / Copy | Debug / serialization | Storage | Capability use | |---|---|---|---|---| @@ -86,7 +88,7 @@ and ordinary serialization therefore do not duplicate a confidential key. | BLAKE3 parallel scratch | Cloned with `Blake3` | Neither | Per-state vectors plus thread-local vectors | Avoid repeated allocation during parallel subtree reduction; keyed modes make stored chaining values secret-derived | | Private-key and AEAD backend transfer records | Private `Clone`/`Copy` only where passed by value or snapshotted by a consuming kernel | Neither | Inline | Fixed-layout, call-scoped handoff to portable, SIMD, or assembly code; no public capability | -## Review Consequences +## Review consequences - `serde` alone covers protocol-visible values. `serde-secrets` is the explicit opt-in for AEAD keys, Ed25519/X25519 secrets, and ML-KEM decapsulation/shared diff --git a/docs/test-vector-coverage.md b/docs/test-vector-coverage.md index 5e9d1909..7ec589a6 100644 --- a/docs/test-vector-coverage.md +++ b/docs/test-vector-coverage.md @@ -1,15 +1,15 @@ -# Test Vector Coverage Ledger +# Test vector coverage ledger -This ledger makes negative-vector coverage visible per primitive instead of -relying on broad claims like "covered by oracles". It is built from the actual -`tests/` and `testdata/` files. +This ledger maps each primitive to its positive oracle, negative behavior +coverage, and known gaps. The entries come from the current `tests/` and +`testdata/` files. Wycheproof suites are vendored from `C2SP/wycheproof` `testvectors_v1/`. The upstream project describes the JSON suites as implementation-agnostic test vectors for known attacks and edge cases, and recommends mapping the vectors to the concrete inputs and outputs of each cryptography API. -## Negative Coverage Policy +## Coverage rules - Prefer Wycheproof when its current JSON suite maps directly to a public rscrypto API. @@ -21,7 +21,7 @@ the concrete inputs and outputs of each cryptography API. negative coverage is limited to parser/format boundaries and dispatch equivalence. -## Coverage Table +## Coverage table | Primitive | Positive vector / oracle coverage | Negative behavior coverage | Gaps / notes | | --- | --- | --- | --- | @@ -31,9 +31,9 @@ the concrete inputs and outputs of each cryptography API. | CRC-64 family | `tests/crc64_properties.rs`; `crc64fast` and `crc-fast` oracles | Property tests cover combine/reset/streaming boundaries | No Wycheproof suite applies | | SHA-224 | `tests/sha2_official_vectors.rs`, `tests/sha256_official_vectors.rs`, `testdata/sha2/sha224.blb` | Streaming/oneshot and dispatch equivalence | No invalid input class | | SHA-256 | `tests/sha2_official_vectors.rs`, `tests/sha256_official_vectors.rs`, `tests/sha256_differential.rs`, `testdata/sha2/sha256.blb` | Streaming/oneshot and dispatch equivalence | Wycheproof has HMAC/HKDF/PBKDF2 SHA-256 suites, not raw SHA-256 | -| SHA-384 | `tests/sha2_official_vectors.rs`, `testdata/sha2/sha384.blb` | Streaming/oneshot and dispatch equivalence | No invalid input class | -| SHA-512 | `tests/sha2_official_vectors.rs`, `tests/sha512_differential.rs`, `testdata/sha2/sha512.blb` | Streaming/oneshot and dispatch equivalence | No invalid input class | -| SHA-512/256 | `tests/sha2_official_vectors.rs`, `testdata/sha2/sha512_256.blb` | Streaming/oneshot and dispatch equivalence | No invalid input class | +| SHA-384 | `tests/sha2_official_vectors.rs`, `testdata/sha2/sha384.blb` | Streaming/oneshot and dispatch equivalence; shared SHA-512-family boundary test panics before overlength absorption | Inputs beyond the FIPS 180-4 length field panic | +| SHA-512 | `tests/sha2_official_vectors.rs`, `tests/sha512_differential.rs`, `testdata/sha2/sha512.blb` | Streaming/oneshot and dispatch equivalence; `message_length_above_boundary_panics` covers the encoded-length boundary | Inputs beyond the FIPS 180-4 length field panic | +| SHA-512/256 | `tests/sha2_official_vectors.rs`, `testdata/sha2/sha512_256.blb` | Streaming/oneshot and dispatch equivalence; shared SHA-512-family boundary test panics before overlength absorption | Inputs beyond the FIPS 180-4 length field panic | | SHA3-224/256/384/512 | `tests/sha3_official_vectors.rs`, `tests/sha3_differential.rs`, `testdata/sha3/sha3_*.blb` | Streaming/oneshot and dispatch equivalence | No invalid input class | | SHAKE128 | `tests/sha3_official_vectors.rs`, `tests/shake128_differential.rs`, `testdata/sha3/shake128.blb` | XOF output-length boundaries | No invalid input class | | SHAKE256 | `tests/sha3_official_vectors.rs`, `tests/shake256_differential.rs`, `testdata/sha3/shake256.blb` | XOF output-length boundaries | No invalid input class | @@ -44,7 +44,7 @@ the concrete inputs and outputs of each cryptography API. | BLAKE2s | `tests/blake2_official_vectors.rs`, `tests/blake2_differential.rs`, `testdata/blake2/blake2s.blb` | Keyed/unkeyed differential coverage | No invalid input class | | BLAKE3 | `tests/blake3_official_vectors.rs`, `tests/blake3_differential.rs`, `testdata/blake3/test_vectors.*` | XOF/keyed/derive-key differential coverage | No invalid input class | | Ascon hash/XOF/CXOF | Final SP 800-232 reference corpora in `tests/ascon_official_vectors.rs`, `tests/ascon_final_kats.rs`, and `testdata/ascon/`; independent coverage in `tests/ascon_hash_oracle.rs` and `tests/ascon_differential.rs` | XOF output and CXOF customization boundaries, streaming, and reset coverage | No invalid input class | -| XXH3 | `tests/xxh3_differential.rs` | Seeded/streaming/property-style differential coverage | Non-cryptographic; no Wycheproof suite applies | +| XXH3 | `tests/xxh3_differential.rs`, `src/hashes/fast/xxh3/stream.rs` unit tests, `fuzz/fuzz_targets/fast_xxh3.rs` | Seeded one-shot properties, streaming oracle/state coverage, and fuzzed partitions | Non-cryptographic; no Wycheproof suite applies | | RapidHash | `tests/rapidhash_differential.rs`; `src/hashes/fast/rapidhash/stream.rs` unit tests | Seeded one-shot property and collection-state schedule differentials; streaming chunk and reset equivalence | Non-cryptographic; no Wycheproof suite applies | | HMAC-SHA-256 | `tests/hmac_sha256_vectors.rs`, `tests/hmac_sha256_proptest.rs`, `tests/hmac_sha2_family_vectors.rs` | `tests/hmac_wycheproof.rs` covers full-tag Wycheproof valid/invalid tags; proptests/family vectors cover mismatch behavior | Truncated-tag Wycheproof groups are out of API scope: `verify_tag` accepts only `[u8; 32]` full tags | | HMAC-SHA-384 | `tests/hmac_sha384_proptest.rs`, `tests/hmac_sha2_family_vectors.rs` | `tests/hmac_wycheproof.rs` covers full-tag Wycheproof valid/invalid tags; proptests/family vectors cover mismatch behavior | Truncated-tag Wycheproof groups are out of API scope: `verify_tag` accepts only `[u8; 48]` full tags | diff --git a/docs/types.md b/docs/types.md index 2cbb2ce1..390e5b17 100644 --- a/docs/types.md +++ b/docs/types.md @@ -1,10 +1,10 @@ -# Public Type Inventory +# Public type inventory -Index of the public `rscrypto` types needed to choose imports, features, and -migration targets. The README keeps the top-level API map; this file carries -the root re-exports and documented module helper types. +Use this inventory to choose imports, feature flags, and migration targets. It +lists root re-exports and documented module helper types; use rustdoc for +method-level details. -Use rustdoc for exhaustive API details. This inventory excludes `diag_*` +This inventory excludes `diag_*` functions, doc-hidden bench hooks, architecture feature constants, and private impl types. @@ -25,7 +25,7 @@ Prelude: `rscrypto::prelude` re-exports `Aead`, `Checksum`, `ChecksumCombine`, `Digest`, `FastHash`, `Kem`, `Mac`, `VerificationError`, and `Xof`. -## Native API Conventions +## API conventions - Prefer caller-provided output buffers and scratch buffers when both forms exist. - Use `alloc` helpers such as `*_to_vec` only when an owned allocation is the right boundary. @@ -56,7 +56,7 @@ Crc24Force, Crc32Config, Crc32Force, Crc64Config, Crc64Force}`, BufferedCrc24OpenPgp, BufferedCrc32, BufferedCrc32C, BufferedCrc64, BufferedCrc64Nvme}`, and `checksum::io::{ChecksumReader, ChecksumWriter}`. -## Cryptographic Hashes +## Cryptographic hashes Features: `crypto-hashes` or `sha2` / `sha3` / `blake2b` / `blake2s` / `blake3` / `ascon-hash`. @@ -80,7 +80,7 @@ Aliases: `hashes::crypto::AsconXof128` and `hashes::crypto::AsconXof128Reader`. `Blake2bKey` and `Blake2sKey` make caller-facing key validation explicit while borrowing key bytes without allocation or copying. -## Fast Hashes +## Fast hashes Features: `fast-hashes` or `xxh3` / `rapidhash`. @@ -100,7 +100,7 @@ All fast hashers use bounded inline state and do not allocate. is reproducible; `RapidRandomState` accepts a fallible entropy callback in pure `no_std` and adds `try_new()` when `getrandom` is enabled. -## MACs & KDFs +## MACs and KDFs Features: `macs` / `kdfs` or `hmac` / `hmac-sha3` / `hkdf` / `pbkdf2` / `kmac` / `poly1305`. @@ -108,12 +108,12 @@ Features: `macs` / `kdfs` or `hmac` / `hmac-sha3` / `hkdf` / `pbkdf2` / `kmac` / |------|------------|----------| | `HmacSha256` / `HmacSha384` / `HmacSha512`; `HmacSha256Tag` / `HmacSha384Tag` / `HmacSha512Tag` | 32-64B | RFC 2104 | | `HmacSha3_224` / `HmacSha3_256` / `HmacSha3_384` / `HmacSha3_512`; matching `HmacSha3_*Tag` types | 28-64B | RFC 2104 over FIPS 202 SHA-3 | -| `Kmac128` / `Kmac256` | variable | SP 800-185 | +| `Kmac128` / `Kmac256` | variable; authentication verification requires 16 / 32 bytes | SP 800-185 | | `Poly1305`, `Poly1305OneTimeKey`, `Poly1305Tag` | 16B tag | RFC 8439 | | `HkdfSha256` / `HkdfSha384` / `HkdfSha512` | 32-64B PRK | RFC 5869 | | `Pbkdf2Sha256` / `Pbkdf2Sha512` | variable | RFC 2898 / SP 800-132 | -## Password Hashing +## Password hashing Features: `password-hashing` or `argon2` / `scrypt` / `phc-strings`. @@ -130,7 +130,7 @@ Password-record operations require `phc-strings`; OS-salted generation also requ PHC parsing and encoding are intentionally internal so attacker-controlled costs cannot bypass the algorithm-specific verification limits. -## Signatures & Key Exchange +## Signatures and key exchange Features: `signatures` / `key-exchange` or `ecdsa` / `ed25519` / `rsa` / `x25519` / `ml-kem`. @@ -165,7 +165,9 @@ RSA public-key verification, import, and caller-filled public encryption require randomized encryption wrappers require `getrandom`. Key generation seeds a key-generation HMAC_DRBG from OS entropy; deterministic caller-supplied salt/blinding APIs remain available for constrained private-operation -integrations that own their entropy boundary. +integrations that own their entropy boundary. Private-key import requires two +conventional half-modulus-width factors that pass trial division and a 32-base +Miller-Rabin probable-prime screen. ML-KEM supports key generation, encapsulation, decapsulation, validated prepared encapsulation keys, and validated prepared decapsulation keys. The core API @@ -201,7 +203,7 @@ must explicitly import `aead::expert::AeadWithNonce` for caller-nonce detached forms remain allocation-free. With `alloc`, decryption has `decrypt_to_vec`; with `alloc` + `getrandom`, sealing has `seal_random_to_vec`. -## Error Types +## Error types | Error | When | Recovery | |-------|------|----------| @@ -212,23 +214,23 @@ detached forms remain allocation-free. With `alloc`, decryption has | `OpenError` | Combined AEAD buffer length is wrong, input is too large, or authentication fails | Correct public lengths; reject opaque verification failures | | `NonceCounterSealError` | AES-GCM nonce counter is exhausted or sealing fails | Rotate the key before counter reuse, or correct the sealing input | | `HkdfOutputLengthError` | HKDF expand exceeds max | Request less output | -| `Pbkdf2Error` | PBKDF2 parameter validation | Adjust iterations / output length | +| `Pbkdf2Error` | PBKDF2 parameter validation fails | Adjust the iteration count, salt, policy, or output length | | `Blake2Error` | Invalid BLAKE2 key or variable output length, or a mismatched streaming output buffer | Correct the public key/output length | | `Argon2Error` | Argon2 configuration, input, entropy, or resource failure | Fix the profile/input or restore resources | | `ScryptError` | scrypt configuration, entropy, or resource failure | Fix N/r/p or restore resources | | `X25519Error` | Low-order DH point | Reject peer key | | `MlKemError` | ML-KEM random source, key, or ciphertext validation failure | Reject input or fix entropy source | -| `RsaKeyError` | RSA DER or component validation failure | Reject key / tighten import policy | +| `RsaKeyError` | RSA DER or component validation fails | Reject the key or tighten the import policy | | `RsaPublicOpError` | RSA public operation input shape/range failure | Fix representative length or reject input | | `RsaPrivateOpError` | RSA private operation, padding, entropy, or fault-check failure | Reject input; do not expose reason to peer | -| `RsaEncryptionError` | RSA public encryption shape, padding, or entropy failure | Fix input / entropy source | -| `RsaKeyGenerationError` | RSA key generation policy or entropy failure | Adjust key size/policy or entropy source | +| `RsaEncryptionError` | RSA public encryption shape, padding, or entropy fails | Fix the input or entropy source | +| `RsaKeyGenerationError` | RSA key-generation policy or entropy fails | Adjust the key size or policy, or fix the entropy source | | `RsaProtocolAlgorithmError` | Unsupported/confused COSE/TLS/X.509 RSA selector | Reject algorithm mapping | | `AsconCxofCustomizationError` | Customization > 256 bytes | Shorten string | | `InvalidHexError` | Hex decode failure | Fix input | | `platform::expert::OverrideError` | Invalid, unsupported, or late detection override | Configure through `platform::expert::try_set_override` before first detection | -## Platform And Dispatch +## Platform and dispatch | Item | Purpose | |------|---------| diff --git a/fuzz-packages/aead-aegis256/Cargo.lock b/fuzz-packages/aead-aegis256/Cargo.lock index 6a52ad44..3a480232 100644 --- a/fuzz-packages/aead-aegis256/Cargo.lock +++ b/fuzz-packages/aead-aegis256/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "aegis" -version = "0.9.12" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e07d39d15384924b35b70d7b8fa1f9a2934101dd3fa4722ede163cc4f9b7b960" +checksum = "58541132f980da31e9aa99f7bdee69bc84bf1e168b9b91ef2dbe8abb7b4ce5dd" dependencies = [ "cc", "softaes", @@ -20,9 +20,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -65,9 +65,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -87,7 +87,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-aead-aegis256" diff --git a/fuzz-packages/aead-aes128gcm/Cargo.lock b/fuzz-packages/aead-aes128gcm/Cargo.lock index b1242ed4..520c1bcb 100644 --- a/fuzz-packages/aead-aes128gcm/Cargo.lock +++ b/fuzz-packages/aead-aes128gcm/Cargo.lock @@ -14,9 +14,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138" +checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" dependencies = [ "cipher", "cpubits", @@ -54,9 +54,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -157,9 +157,9 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -185,9 +185,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -201,9 +201,9 @@ dependencies = [ [[package]] name = "polyval" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b20f20e954175de5f463f67781b35583397d916b1d148738923711b2ad16bee8" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" dependencies = [ "cpubits", "cpufeatures", @@ -218,7 +218,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-aead-aes128gcm" diff --git a/fuzz-packages/aead-aes128gcmsiv/Cargo.lock b/fuzz-packages/aead-aes128gcmsiv/Cargo.lock index 5f01fc71..7e8bc8ba 100644 --- a/fuzz-packages/aead-aes128gcmsiv/Cargo.lock +++ b/fuzz-packages/aead-aes128gcmsiv/Cargo.lock @@ -46,9 +46,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -188,7 +188,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-aead-aes128gcmsiv" diff --git a/fuzz-packages/aead-aes256gcm/Cargo.lock b/fuzz-packages/aead-aes256gcm/Cargo.lock index a99cee57..5ef7f9e0 100644 --- a/fuzz-packages/aead-aes256gcm/Cargo.lock +++ b/fuzz-packages/aead-aes256gcm/Cargo.lock @@ -14,9 +14,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138" +checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" dependencies = [ "cipher", "cpubits", @@ -54,9 +54,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -157,9 +157,9 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -185,9 +185,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -201,9 +201,9 @@ dependencies = [ [[package]] name = "polyval" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b20f20e954175de5f463f67781b35583397d916b1d148738923711b2ad16bee8" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" dependencies = [ "cpubits", "cpufeatures", @@ -218,7 +218,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-aead-aes256gcm" diff --git a/fuzz-packages/aead-aes256gcmsiv/Cargo.lock b/fuzz-packages/aead-aes256gcmsiv/Cargo.lock index f287607e..dfa5d6a3 100644 --- a/fuzz-packages/aead-aes256gcmsiv/Cargo.lock +++ b/fuzz-packages/aead-aes256gcmsiv/Cargo.lock @@ -46,9 +46,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -188,7 +188,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-aead-aes256gcmsiv" diff --git a/fuzz-packages/aead-ascon128/Cargo.lock b/fuzz-packages/aead-ascon128/Cargo.lock index 4bdc36c8..15bc01d0 100644 --- a/fuzz-packages/aead-ascon128/Cargo.lock +++ b/fuzz-packages/aead-ascon128/Cargo.lock @@ -38,9 +38,9 @@ checksum = "c90c8493a0b3bf8678a82f096821a2c527dd6344bbcef6265bfcc09745f44e24" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -82,9 +82,9 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -110,9 +110,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -132,7 +132,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-aead-ascon128" diff --git a/fuzz-packages/aead-chacha20poly1305/Cargo.lock b/fuzz-packages/aead-chacha20poly1305/Cargo.lock index a7533f8c..cd2ff2a1 100644 --- a/fuzz-packages/aead-chacha20poly1305/Cargo.lock +++ b/fuzz-packages/aead-chacha20poly1305/Cargo.lock @@ -29,9 +29,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -131,9 +131,9 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -159,9 +159,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -191,7 +191,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-aead-chacha20poly1305" diff --git a/fuzz-packages/aead-nonce-counter/Cargo.lock b/fuzz-packages/aead-nonce-counter/Cargo.lock index d49d9a4e..5b5f25ce 100644 --- a/fuzz-packages/aead-nonce-counter/Cargo.lock +++ b/fuzz-packages/aead-nonce-counter/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -55,9 +55,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -77,7 +77,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-aead-nonce-counter" diff --git a/fuzz-packages/aead-xchacha20poly1305/Cargo.lock b/fuzz-packages/aead-xchacha20poly1305/Cargo.lock index 914f292f..3b19f3f1 100644 --- a/fuzz-packages/aead-xchacha20poly1305/Cargo.lock +++ b/fuzz-packages/aead-xchacha20poly1305/Cargo.lock @@ -29,9 +29,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -131,9 +131,9 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -159,9 +159,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -191,7 +191,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-aead-xchacha20poly1305" diff --git a/fuzz-packages/auth-argon2/Cargo.lock b/fuzz-packages/auth-argon2/Cargo.lock index efaa070d..8fb4467c 100644 --- a/fuzz-packages/auth-argon2/Cargo.lock +++ b/fuzz-packages/auth-argon2/Cargo.lock @@ -46,9 +46,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -125,9 +125,9 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -144,9 +144,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -185,7 +185,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-auth-argon2" diff --git a/fuzz-packages/auth-ed25519/Cargo.lock b/fuzz-packages/auth-ed25519/Cargo.lock index fcb285a7..013bb2aa 100644 --- a/fuzz-packages/auth-ed25519/Cargo.lock +++ b/fuzz-packages/auth-ed25519/Cargo.lock @@ -25,9 +25,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -90,7 +90,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -193,9 +193,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -219,18 +219,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -252,7 +252,7 @@ dependencies = [ [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-auth-ed25519" @@ -288,31 +288,31 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", ] [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -359,9 +359,20 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", diff --git a/fuzz-packages/auth-hkdf/Cargo.lock b/fuzz-packages/auth-hkdf/Cargo.lock index a38986a5..c4719c32 100644 --- a/fuzz-packages/auth-hkdf/Cargo.lock +++ b/fuzz-packages/auth-hkdf/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -123,9 +123,9 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -142,9 +142,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -164,7 +164,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-auth-hkdf" diff --git a/fuzz-packages/auth-hmac/Cargo.lock b/fuzz-packages/auth-hmac/Cargo.lock index e59264d5..e14dfe58 100644 --- a/fuzz-packages/auth-hmac/Cargo.lock +++ b/fuzz-packages/auth-hmac/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -114,9 +114,9 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -133,9 +133,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -155,7 +155,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-auth-hmac" diff --git a/fuzz-packages/auth-kmac256/Cargo.lock b/fuzz-packages/auth-kmac256/Cargo.lock index 54ceb163..ed5bce65 100644 --- a/fuzz-packages/auth-kmac256/Cargo.lock +++ b/fuzz-packages/auth-kmac256/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -61,9 +61,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -83,7 +83,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-auth-kmac256" diff --git a/fuzz-packages/auth-kmac256/corpus/auth_kmac256/seed-exact-key-bytepad b/fuzz-packages/auth-kmac256/corpus/auth_kmac256/seed-exact-key-bytepad new file mode 100644 index 00000000..63a58009 --- /dev/null +++ b/fuzz-packages/auth-kmac256/corpus/auth_kmac256/seed-exact-key-bytepad @@ -0,0 +1 @@ +!z?0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0 diff --git a/fuzz-packages/auth-kmac256/corpus/auth_kmac256/seed-short-auth-tag b/fuzz-packages/auth-kmac256/corpus/auth_kmac256/seed-short-auth-tag new file mode 100644 index 00000000..5e0778e5 --- /dev/null +++ b/fuzz-packages/auth-kmac256/corpus/auth_kmac256/seed-short-auth-tag @@ -0,0 +1,2 @@ +rs +0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef diff --git a/fuzz-packages/auth-pbkdf2/Cargo.lock b/fuzz-packages/auth-pbkdf2/Cargo.lock index 788479a7..a661e37c 100644 --- a/fuzz-packages/auth-pbkdf2/Cargo.lock +++ b/fuzz-packages/auth-pbkdf2/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -114,9 +114,9 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -133,9 +133,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -165,7 +165,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-auth-pbkdf2" diff --git a/fuzz-packages/auth-phc/Cargo.lock b/fuzz-packages/auth-phc/Cargo.lock index 4f316416..0aac3f41 100644 --- a/fuzz-packages/auth-phc/Cargo.lock +++ b/fuzz-packages/auth-phc/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -55,9 +55,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" diff --git a/fuzz-packages/auth-scrypt/Cargo.lock b/fuzz-packages/auth-scrypt/Cargo.lock index e4c51e68..5a7deb35 100644 --- a/fuzz-packages/auth-scrypt/Cargo.lock +++ b/fuzz-packages/auth-scrypt/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -118,9 +118,9 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -146,9 +146,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -178,7 +178,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-auth-scrypt" diff --git a/fuzz-packages/auth-x25519/Cargo.lock b/fuzz-packages/auth-x25519/Cargo.lock index bb4ec7ca..4267b661 100644 --- a/fuzz-packages/auth-x25519/Cargo.lock +++ b/fuzz-packages/auth-x25519/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -58,7 +58,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -96,9 +96,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -112,18 +112,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -142,7 +142,7 @@ checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-auth-x25519" @@ -178,9 +178,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -188,22 +188,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -220,9 +220,20 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -264,5 +275,5 @@ checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] diff --git a/fuzz-packages/checksums-crc16/Cargo.lock b/fuzz-packages/checksums-crc16/Cargo.lock index 39682f01..1c1a2e88 100644 --- a/fuzz-packages/checksums-crc16/Cargo.lock +++ b/fuzz-packages/checksums-crc16/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -70,9 +70,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -92,7 +92,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-checksums-crc16" diff --git a/fuzz-packages/checksums-crc24/Cargo.lock b/fuzz-packages/checksums-crc24/Cargo.lock index 0490bdd1..b8a0bfa6 100644 --- a/fuzz-packages/checksums-crc24/Cargo.lock +++ b/fuzz-packages/checksums-crc24/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -70,9 +70,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -92,7 +92,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-checksums-crc24" diff --git a/fuzz-packages/checksums-crc32/Cargo.lock b/fuzz-packages/checksums-crc32/Cargo.lock index 45c6e0b2..76ed2839 100644 --- a/fuzz-packages/checksums-crc32/Cargo.lock +++ b/fuzz-packages/checksums-crc32/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -70,9 +70,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -92,7 +92,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-checksums-crc32" diff --git a/fuzz-packages/checksums-crc64/Cargo.lock b/fuzz-packages/checksums-crc64/Cargo.lock index cc98234d..c910382d 100644 --- a/fuzz-packages/checksums-crc64/Cargo.lock +++ b/fuzz-packages/checksums-crc64/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -70,9 +70,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -92,7 +92,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-checksums-crc64" diff --git a/fuzz-packages/fast-rapidhash/Cargo.lock b/fuzz-packages/fast-rapidhash/Cargo.lock index 539a7041..0f99f2e3 100644 --- a/fuzz-packages/fast-rapidhash/Cargo.lock +++ b/fuzz-packages/fast-rapidhash/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -55,9 +55,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" diff --git a/fuzz-packages/fast-xxh3/Cargo.lock b/fuzz-packages/fast-xxh3/Cargo.lock index 2f49b2a4..207a69f6 100644 --- a/fuzz-packages/fast-xxh3/Cargo.lock +++ b/fuzz-packages/fast-xxh3/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -55,9 +55,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -77,7 +77,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-fast-xxh3" @@ -104,6 +104,6 @@ checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "xxhash-rust" -version = "0.8.16" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" diff --git a/fuzz-packages/hash-ascon/Cargo.lock b/fuzz-packages/hash-ascon/Cargo.lock index f8a049e1..946b3949 100644 --- a/fuzz-packages/hash-ascon/Cargo.lock +++ b/fuzz-packages/hash-ascon/Cargo.lock @@ -35,9 +35,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -89,9 +89,9 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -108,9 +108,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -130,7 +130,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-hash-ascon" diff --git a/fuzz-packages/hash-blake2/Cargo.lock b/fuzz-packages/hash-blake2/Cargo.lock index 74d5673c..d993c23c 100644 --- a/fuzz-packages/hash-blake2/Cargo.lock +++ b/fuzz-packages/hash-blake2/Cargo.lock @@ -28,9 +28,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -107,9 +107,9 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -126,9 +126,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -148,7 +148,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-hash-blake2" diff --git a/fuzz-packages/hash-blake3/Cargo.lock b/fuzz-packages/hash-blake3/Cargo.lock index d1477e46..61858c38 100644 --- a/fuzz-packages/hash-blake3/Cargo.lock +++ b/fuzz-packages/hash-blake3/Cargo.lock @@ -36,9 +36,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -96,9 +96,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -118,7 +118,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-hash-blake3" diff --git a/fuzz-packages/hash-sha2/Cargo.lock b/fuzz-packages/hash-sha2/Cargo.lock index 20e3179c..b66776e7 100644 --- a/fuzz-packages/hash-sha2/Cargo.lock +++ b/fuzz-packages/hash-sha2/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -89,9 +89,9 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -108,9 +108,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -130,7 +130,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-hash-sha2" diff --git a/fuzz-packages/hash-sha3/Cargo.lock b/fuzz-packages/hash-sha3/Cargo.lock index c7628bc5..23941a0e 100644 --- a/fuzz-packages/hash-sha3/Cargo.lock +++ b/fuzz-packages/hash-sha3/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -65,6 +65,17 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "cshake" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6250a2d96a09edbe8e75ed29c87d05512ee2cbb24c7e8c684657f7930ffd3c6" +dependencies = [ + "digest", + "keccak", + "sponge-cursor", +] + [[package]] name = "digest" version = "0.11.3" @@ -95,9 +106,9 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -124,9 +135,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -146,12 +157,13 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-hash-sha3" version = "0.0.0" dependencies = [ + "cshake", "digest", "libfuzzer-sys", "rscrypto", diff --git a/fuzz-packages/hash-sha3/Cargo.toml b/fuzz-packages/hash-sha3/Cargo.toml index 7734a909..b7ed0586 100644 --- a/fuzz-packages/hash-sha3/Cargo.toml +++ b/fuzz-packages/hash-sha3/Cargo.toml @@ -8,12 +8,13 @@ edition = "2024" cargo-fuzz = true [dependencies] +cshake = { version = "0.2.1", default-features = false } +digest = "0.11" libfuzzer-sys = "0.4" rscrypto = { path = "../..", features = ["std", "sha3"] } rscrypto_fuzz = { package = "rscrypto-fuzz-support", path = "../../fuzz/support" } -digest = "0.11" sha3 = "0.12" -tiny-keccak = { version = "2.0", features = ["cshake", "shake"] } +tiny-keccak = { version = "2.0", features = ["shake"] } [workspace] members = ["."] diff --git a/fuzz-packages/hash-sha3/corpus/hash_cshake256/seed-bytepad-exact-rate b/fuzz-packages/hash-sha3/corpus/hash_cshake256/seed-bytepad-exact-rate new file mode 100644 index 00000000..33c105fc --- /dev/null +++ b/fuzz-packages/hash-sha3/corpus/hash_cshake256/seed-bytepad-exact-rate @@ -0,0 +1 @@ +rsc@y0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456 diff --git a/fuzz-packages/surface-hex-parse/Cargo.lock b/fuzz-packages/surface-hex-parse/Cargo.lock index c9a9f528..722ecdf9 100644 --- a/fuzz-packages/surface-hex-parse/Cargo.lock +++ b/fuzz-packages/surface-hex-parse/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -55,9 +55,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -77,7 +77,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-support" diff --git a/fuzz-packages/traits-io/Cargo.lock b/fuzz-packages/traits-io/Cargo.lock index 518310b1..0a4d28bb 100644 --- a/fuzz-packages/traits-io/Cargo.lock +++ b/fuzz-packages/traits-io/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -55,9 +55,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -77,7 +77,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-fuzz-support" diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index b6ac9b68..73ccdeee 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -24,9 +24,9 @@ dependencies = [ [[package]] name = "aegis" -version = "0.9.12" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e07d39d15384924b35b70d7b8fa1f9a2934101dd3fa4722ede163cc4f9b7b960" +checksum = "58541132f980da31e9aa99f7bdee69bc84bf1e168b9b91ef2dbe8abb7b4ce5dd" dependencies = [ "cc", "softaes", @@ -45,9 +45,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138" +checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" dependencies = [ "cipher 0.5.2", "cpubits", @@ -61,7 +61,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" dependencies = [ "aead 0.6.1", - "aes 0.9.1", + "aes 0.9.2", "cipher 0.5.2", "ctr 0.10.1", "ghash", @@ -208,9 +208,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -386,6 +386,17 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "cshake" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6250a2d96a09edbe8e75ed29c87d05512ee2cbb24c7e8c684657f7930ffd3c6" +dependencies = [ + "digest 0.11.3", + "keccak", + "sponge-cursor", +] + [[package]] name = "ctr" version = "0.9.2" @@ -438,7 +449,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -649,7 +660,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ - "polyval 0.7.2", + "polyval 0.7.3", ] [[package]] @@ -703,9 +714,9 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "subtle", "typenum", @@ -752,9 +763,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -885,9 +896,9 @@ dependencies = [ [[package]] name = "polyval" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b20f20e954175de5f463f67781b35583397d916b1d148738923711b2ad16bee8" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" dependencies = [ "cpubits", "cpufeatures 0.3.0", @@ -932,18 +943,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -1016,6 +1027,7 @@ dependencies = [ "blake3", "chacha20poly1305", "crc", + "cshake", "digest 0.11.3", "ed25519-dalek", "hkdf", @@ -1115,9 +1127,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -1125,22 +1137,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -1252,9 +1264,20 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -1339,9 +1362,9 @@ dependencies = [ [[package]] name = "xxhash-rust" -version = "0.8.16" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "zeroize" @@ -1360,5 +1383,5 @@ checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index b147a535..ce331f94 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -47,10 +47,11 @@ x25519-dalek = { version = "2.0.1", features = ["static_secrets"] } ascon-hash = "0.4" blake2 = "0.11.0-rc.6" blake3 = "1.8" +cshake = { version = "0.2.1", default-features = false } digest = "0.11" sha2 = "0.11" sha3 = "0.12" -tiny-keccak = { version = "2.0", features = ["cshake", "kmac", "shake"] } +tiny-keccak = { version = "2.0", features = ["kmac", "shake"] } # Fast hashes rapidhash = "=4.5.1" diff --git a/fuzz/corpus/auth_kmac256/seed-exact-key-bytepad b/fuzz/corpus/auth_kmac256/seed-exact-key-bytepad new file mode 100644 index 00000000..63a58009 --- /dev/null +++ b/fuzz/corpus/auth_kmac256/seed-exact-key-bytepad @@ -0,0 +1 @@ +!z?0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0 diff --git a/fuzz/corpus/auth_kmac256/seed-short-auth-tag b/fuzz/corpus/auth_kmac256/seed-short-auth-tag new file mode 100644 index 00000000..5e0778e5 --- /dev/null +++ b/fuzz/corpus/auth_kmac256/seed-short-auth-tag @@ -0,0 +1,2 @@ +rs +0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef diff --git a/fuzz/corpus/hash_cshake256/seed-bytepad-exact-rate b/fuzz/corpus/hash_cshake256/seed-bytepad-exact-rate new file mode 100644 index 00000000..33c105fc --- /dev/null +++ b/fuzz/corpus/hash_cshake256/seed-bytepad-exact-rate @@ -0,0 +1 @@ +rsc@y0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456 diff --git a/fuzz/target_impls/auth_kmac256.rs b/fuzz/target_impls/auth_kmac256.rs index 1fdd20fc..31854463 100644 --- a/fuzz/target_impls/auth_kmac256.rs +++ b/fuzz/target_impls/auth_kmac256.rs @@ -1,6 +1,20 @@ use rscrypto::Kmac256; use rscrypto_fuzz::{FuzzInput, some_or_return, split_at_ratio}; +fn encoded_string_len(len: usize) -> usize { + let bits = len.strict_mul(8); + let width = ((usize::BITS - bits.leading_zeros()) as usize).div_ceil(8).max(1); + 1usize.strict_add(width).strict_add(len) +} + +fn bytepad_is_aligned(rate: usize, segments: &[usize]) -> bool { + let encoded_len = segments + .iter() + .map(|&len| encoded_string_len(len)) + .fold(2usize, usize::strict_add); + encoded_len.is_multiple_of(rate) +} + pub fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); @@ -31,11 +45,30 @@ pub fn run(data: &[u8]) { kmac.finalize_into(&mut reset_out); assert_eq!(expected, reset_out, "kmac changed after reset"); - // Property: verify accepts correct tag - Kmac256::verify_tag(key, custom, message, &expected).expect("verify must accept correct tag"); + // Property: the authentication API enforces its documented strength floor, + // while the primitive API accepts every nonempty KMAC output. + assert_eq!( + Kmac256::verify_tag(key, custom, message, &expected).is_ok(), + out_len >= Kmac256::MIN_AUTH_TAG_SIZE, + "kmac256 authentication policy mismatch" + ); + Kmac256::verify_tag_primitive(key, custom, message, &expected) + .expect("primitive verification must accept a correct nonempty output"); + + let mut corrupted = expected.clone(); + corrupted[out_len / 2] ^= 1; + assert!( + Kmac256::verify_tag_primitive(key, custom, message, &corrupted).is_err(), + "primitive verification accepted a corrupted output" + ); + assert!( + Kmac256::verify_tag(key, custom, message, &corrupted).is_err(), + "authentication verification accepted a corrupted tag" + ); - // Differential: rscrypto ↔ tiny-keccak - { + // tiny-keccak 2.0.2 mishandles an exactly full SP 800-185 bytepad block. + // Independent fixed vectors cover those boundaries in the integration tests. + if !bytepad_is_aligned(136, &[4, custom.len()]) && !bytepad_is_aligned(136, &[key.len()]) { use tiny_keccak::{Hasher, Kmac as OracleKmac}; let mut oracle = OracleKmac::v256(key, custom); diff --git a/fuzz/target_impls/hash_cshake256.rs b/fuzz/target_impls/hash_cshake256.rs index 7dc24015..5ebdc605 100644 --- a/fuzz/target_impls/hash_cshake256.rs +++ b/fuzz/target_impls/hash_cshake256.rs @@ -19,14 +19,18 @@ pub fn run(data: &[u8]) { out_len.strict_mul(squeeze_split as usize) / 255 }; - // Differential: rscrypto ↔ tiny-keccak + // Differential: rscrypto ↔ RustCrypto cshake let mut expected = vec![0u8; out_len]; - use tiny_keccak::{CShake, Hasher, Xof as _}; + use cshake::{ + CShake256 as OracleCshake256, + digest::{ExtendableOutput as _, Update as _, XofReader as _}, + }; - let mut oracle = CShake::v256(name, custom); + let mut oracle = OracleCshake256::new_with_function_name(name, custom); oracle.update(message); - oracle.squeeze(&mut expected[..split_out]); - oracle.squeeze(&mut expected[split_out..]); + let mut oracle_reader = oracle.finalize_xof(); + oracle_reader.read(&mut expected[..split_out]); + oracle_reader.read(&mut expected[split_out..]); let (a, b) = split_at_ratio(message, message_split); let mut h2 = Cshake256::new(name, custom); diff --git a/justfile b/justfile index 27bcbd35..dae2c4b8 100644 --- a/justfile +++ b/justfile @@ -142,6 +142,9 @@ test-miri *crates="": test-rsa-leakage: @scripts/test/test-rsa-leakage.sh +test-rsa-macos-asm: + @scripts/test/test-rsa-macos-asm.sh + test-fuzz *args="": @scripts/test/test-fuzz.sh {{ args }} diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 84fc7922..f3cb4461 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "nightly-2026-04-27" +channel = "nightly-2026-07-17" profile = "default" components = ["clippy", "rustfmt", "miri", "rust-src"] diff --git a/scripts/README.md b/scripts/README.md index 000ffb0f..42d4a02d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -12,6 +12,10 @@ the recipes reported by `just --list`. | `check/check-all.sh` | `just check-all` | | `check/check-feature-matrix.sh`| `just check-feature-matrix`, `scripts/check/check.sh`, `ci/run-rust-job.sh` | | `check/asm-ledger.sh` | `scripts/check/check.sh` | +| `check/rsa-asm-provenance.sh` | `check/asm-ledger.sh`; direct `--archive PATH` reconstructs the three pinned RSA snapshots offline | +| `check/signature-asm-provenance.py` | `check/asm-ledger.sh`; direct `--upstream-repo PATH [--clang PATH]` reproduces the 36 pinned ECDSA, Ed25519, and X25519 snapshots | +| `check/hash-vector-provenance.py` | `scripts/check/check.sh`; optional exact upstream checkouts reproduce hash-vector corpora | +| `check/auth-vector-provenance.py` | `scripts/check/check.sh`; optional `--upstream-root PATH` reproduces the pinned C2SP/Wycheproof corpus | | `check/zeroize-evidence.sh` | `just check-zeroize-evidence`, `scripts/check/check-all.sh` | | `ci/ci-check.sh` | `just ci-check`, `ci/run-rust-job.sh` | | `ci/native-check.sh` | `ci/run-rust-job.sh` | @@ -21,6 +25,7 @@ the recipes reported by `just --list`. | `test/test-fuzz.sh` | `just test-fuzz`, `ci/run-rust-job.sh` | | `test/test-fuzz-asan.sh` | `just test-fuzz-asan`, `ci/run-rust-job.sh` | | `test/test-rsa-leakage.sh` | `just test-rsa-leakage`, `ci/run-rust-job.sh` | +| `test/test-rsa-macos-asm.sh` | `just test-rsa-macos-asm` on a physical local Apple Silicon Mac | | `test/test-coverage.sh` | `just test-coverage`, `just test-fuzz-coverage`, `weekly.yaml` | | `bench/bench.sh` | `just bench`, `just bench-quick` | | `ci/check-action-pins.sh` | `just check-actions`, `ci/ci-check.sh`, `ci/dependabot-smoke.sh` | @@ -51,6 +56,7 @@ the recipes reported by `just --list`. | `ci/pre-push.sh` | `just push`, `just push-full` | | `ct/artifacts.sh` | `just ct`, `just ct-artifacts`, `scripts/ct/full.py` | | `ct/dudect.sh` | `just ct-dudect`, `scripts/ct/full.py` | +| `ct/dudect_report_test.py` | `scripts/check/check.sh` | | `ct/python.sh` | CT recipes, `ci/run-rust-job.sh`, and Python-backed CT, check, and release scripts | | `update/update-all.sh` | `just update`, `just update-check` | @@ -113,13 +119,16 @@ Package-manager tools install into a fresh runner-temporary root; CI never restores Cargo binaries, Cargo install metadata, Go module state, or OPAM switches from a cache. Cargo installs exact crates from crates.io and authenticates crate contents against registry checksums. Go installs an exact -module through the public checksum database. Ubuntu 24.04 APT packages are -exact-versioned and authenticated by signed repository metadata. OPAM uses -exact packages from a repository pinned to a full Git commit and verifies -package source hashes from that immutable metadata. Rustup receives only the -exact toolchain declared in `rust-toolchain.toml`; runner images must provide -rustup, which verifies component downloads against the exact distribution -manifest, because network bootstrap installers are rejected. +module through the public checksum database. Ubuntu 24.04 APT dependencies +resolve from signed repository metadata; installation pins each signed +candidate selected after the metadata refresh, verifies the installed version, +and refuses downgrades. OPAM uses exact packages from a repository pinned to a +full Git commit and verifies package source hashes from that immutable metadata. +CT formal reports bind the resulting BINSEC executable by SHA-256. Rustup +receives only the exact toolchain declared in `rust-toolchain.toml`; runner +images must provide rustup, which verifies component downloads against the +exact distribution manifest, because network bootstrap installers are +rejected. ## Results layout diff --git a/scripts/bench/bench.sh b/scripts/bench/bench.sh index 622a4c3d..95833743 100755 --- a/scripts/bench/bench.sh +++ b/scripts/bench/bench.sh @@ -169,6 +169,11 @@ RUN_ARCH="$(detect_bench_arch)" RUN_COMMIT="$(git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null || echo unknown)" RUN_MODE="${RSCRYPTO_BENCH_MODE:-local}" +if [[ "$RUN_MODE" == "local" && "$RUN_OS" == "macos" \ + && -z "${RUSTFLAGS+x}" && -z "${CARGO_ENCODED_RUSTFLAGS+x}" ]]; then + export RUSTFLAGS="-C target-cpu=native" +fi + if [[ "$RUN_MODE" == "local" ]]; then RUN_DATE="$(date +"%Y-%m-%d")" RUN_TIME="$(date +"%H_%M_%S")" diff --git a/scripts/check/asm-ledger.sh b/scripts/check/asm-ledger.sh index f1fa9e5d..28a651bd 100755 --- a/scripts/check/asm-ledger.sh +++ b/scripts/check/asm-ledger.sh @@ -4,6 +4,9 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +"$SCRIPT_DIR/rsa-asm-provenance.sh" +"$SCRIPT_DIR/../ct/python.sh" "$SCRIPT_DIR/signature-asm-provenance.py" + TMP_DIR="$(mktemp -d)" trap 'rm -rf "$TMP_DIR"' EXIT diff --git a/scripts/check/auth-vector-provenance.py b/scripts/check/auth-vector-provenance.py new file mode 100644 index 00000000..1bc778ea --- /dev/null +++ b/scripts/check/auth-vector-provenance.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Verify the pinned authentication Wycheproof corpus.""" + +from __future__ import annotations + +import argparse +import hashlib +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +LOCAL_DIR = ROOT / "testdata/auth/wycheproof" +UPSTREAM_COMMIT = "b61843a9a5115bb758134b6a1f5d5e502d445342" +FILES = { + "ecdsa_secp256r1_sha256_test.json": "182db4f3e230f6f9fa9f800d2a614dede30284b8e8438bbfe1171905402e9332", + "ecdsa_secp384r1_sha384_test.json": "8a5b3ae1760975143414811f13588c24d951d9d8c904195087ba327591dfe9cc", + "ed25519_test.json": "70471c053c711731f2195ef4875b60ea7f5d6793939d99058ac12da810cb8e00", + "hkdf_sha256_test.json": "bb2b462a38b251cb52a2aede706d6d4b62b26864f4e80c95497507ddb07c5f1e", + "hkdf_sha384_test.json": "69ff6ea3657bb9c1b8cdffbbb4e7832353d08fd15c0d9997b03f7a6b180e3678", + "hmac_sha256_test.json": "2d201cfa61d1bf95e6f5d07d96634b4a348b31e8eaa277ad7c8d09677b7a743f", + "hmac_sha384_test.json": "28b9776e979dd755d852ca471043ea6cedce8b15f7a28abdf6ea9efd982b43c0", + "hmac_sha512_test.json": "b6c90477bdb4a6fc8ee3d1f7b2c0b69a8dfffab34718abaa6cabd71cc2ba1207", + "kmac256_no_customization_test.json": "950b9e8f64bd4e614aa3d825f0cd0570ec33c6cdfc81cbd043c429265149a671", + "pbkdf2_hmacsha256_test.json": "fa21062c95e385aab1714d607c320d534c75082f9594bdf965dba3d934fd17ef", + "pbkdf2_hmacsha512_test.json": "3bc72b80f5c3d79cc2565b9b98dd982e7b1e1082df3a356d648a4d77470aa1d7", + "x25519_test.json": "35c3f5231cf25cc640b524d403461deee9e49441d5d915a3a25b2c8ff5adbe7d", +} + + +def fail(message: str) -> None: + raise SystemExit(f"auth-vector-provenance: {message}") + + +def read_pinned(path: Path, expected: str) -> bytes: + try: + data = path.read_bytes() + except OSError as error: + fail(f"cannot read {path}: {error}") + actual = hashlib.sha256(data).hexdigest() + if actual != expected: + fail(f"{path} has SHA-256 {actual}; expected {expected}") + return data + + +def require_exact_checkout(root: Path) -> None: + try: + actual = subprocess.run( + ["git", "-C", str(root), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError) as error: + fail(f"cannot inspect upstream checkout {root}: {error}") + if actual != UPSTREAM_COMMIT: + fail(f"{root} is at {actual}; expected {UPSTREAM_COMMIT}") + + +def read_upstream_blob(root: Path, name: str, expected: str) -> bytes: + try: + data = subprocess.run( + ["git", "-C", str(root), "show", f"{UPSTREAM_COMMIT}:testvectors_v1/{name}"], + check=True, + capture_output=True, + ).stdout + except (OSError, subprocess.CalledProcessError) as error: + fail(f"cannot read pinned upstream blob testvectors_v1/{name}: {error}") + actual = hashlib.sha256(data).hexdigest() + if actual != expected: + fail(f"upstream {name} has SHA-256 {actual}; expected {expected}") + return data + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--upstream-root", type=Path) + args = parser.parse_args() + + actual_files = {path.name for path in LOCAL_DIR.glob("*.json")} + expected_files = set(FILES) + if actual_files != expected_files: + missing = sorted(expected_files - actual_files) + extra = sorted(actual_files - expected_files) + fail(f"corpus coverage drift; missing={missing}, extra={extra}") + + for name, expected in FILES.items(): + local = read_pinned(LOCAL_DIR / name, expected) + if args.upstream_root is not None: + upstream = read_upstream_blob(args.upstream_root, name, expected) + if upstream != local: + fail(f"{name} differs from pinned upstream bytes") + + if args.upstream_root is not None: + require_exact_checkout(args.upstream_root) + + print("auth-vector-provenance: committed corpus matches the pinned C2SP/Wycheproof commit") + + +if __name__ == "__main__": + main() diff --git a/scripts/check/check.sh b/scripts/check/check.sh index 7f154a8a..c9995b5d 100755 --- a/scripts/check/check.sh +++ b/scripts/check/check.sh @@ -72,6 +72,22 @@ if ! "$SCRIPT_DIR/asm-ledger.sh" >"$LOG_DIR/asm-ledger.log" 2>&1; then fi ok +step "Checking hash vector provenance" +if ! "$SCRIPT_DIR/../ct/python.sh" "$SCRIPT_DIR/hash-vector-provenance.py" >"$LOG_DIR/hash-vectors.log" 2>&1; then + fail + show_error "$LOG_DIR/hash-vectors.log" + exit 1 +fi +ok + +step "Checking authentication vector provenance" +if ! "$SCRIPT_DIR/../ct/python.sh" "$SCRIPT_DIR/auth-vector-provenance.py" >"$LOG_DIR/auth-vectors.log" 2>&1; then + fail + show_error "$LOG_DIR/auth-vectors.log" + exit 1 +fi +ok + step "Checking CT assembly scanner" if ! "$SCRIPT_DIR/../ct/python.sh" "$SCRIPT_DIR/../ct/asm_heuristics_test.py" >"$LOG_DIR/ct-asm-scanner.log" 2>&1; then fail @@ -80,6 +96,14 @@ if ! "$SCRIPT_DIR/../ct/python.sh" "$SCRIPT_DIR/../ct/asm_heuristics_test.py" >" fi ok +step "Checking DudeCT evidence parsing" +if ! "$SCRIPT_DIR/../ct/python.sh" "$SCRIPT_DIR/../ct/dudect_report_test.py" >"$LOG_DIR/ct-dudect-report.log" 2>&1; then + fail + show_error "$LOG_DIR/ct-dudect-report.log" + exit 1 +fi +ok + step "Checking CT evidence validation" if ! "$SCRIPT_DIR/../ct/python.sh" "$SCRIPT_DIR/../ct/evidence_validation_test.py" >"$LOG_DIR/ct-evidence-validation.log" 2>&1; then fail diff --git a/scripts/check/hash-vector-provenance.py b/scripts/check/hash-vector-provenance.py new file mode 100755 index 00000000..160e93ab --- /dev/null +++ b/scripts/check/hash-vector-provenance.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""Verify pinned hash-vector corpora and their deterministic transforms.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + +SHA2_COMMIT = "82c36a428f8d6f05f3bfccdedb243e9d1f85359d" +SHA3_COMMIT = "1637e892b5658941d04a4d895165b66780c7d7ab" +BLAKE2_COMMIT = "ed1974ea83433eba7b2d95c5dcd9ac33cb847913" +BLAKE3_COMMIT = "8aa5145039b972ba30e98e788752d37d14568824" + +SHA2_FILES = { + "sha224.blb": "59b185972521af418fd49a079de3d5f5bed74cd76d80473da51cab3faee6c7d0", + "sha256.blb": "bb096934bb7e43e41ce143d211397afca6fcdfe243a39811688ea31aae6f800a", + "sha384.blb": "e8fe66c07ba336fae2c0aa4c87cb768f41bd4ed318ee1a36fbde0a68581946ec", + "sha512.blb": "1cc0e86571f2f4e3bc81438ce7b6c25c118d2d7437355240113f59cbb782c8d6", + "sha512_256.blb": "95195b758e362d92ff0cebebac4cca696512ea5811b635243bc70e29164e5786", +} + +SHA3_FILES = { + "sha3_224.blb": "9c6676da06e149cf2f71be4b4554d042f7c5fa6d5f43696a30ca8d6747c85a23", + "sha3_256.blb": "00e7834e0abc16614b772a0c6245a29c16807e79c54aa153b008f11cd26268d6", + "sha3_384.blb": "cd4d9c607c5518a0274415b89512ea4cb9be3fd25edf8269aef566d904b797e9", + "sha3_512.blb": "f551f332df7fc50b313544aadad361ce3ce5fd91f21259b93c64b35157904be3", + "shake128.blb": "5900de7f0e09bfd290bee04b183f69fef8407a022491f5f6018cad737de53e4a", + "shake256.blb": "4b65535c6e28e34f840df71b6dd0d99f51bac13d191e3769861e8560bf9d2373", +} + +BLAKE2_SOURCE_SHA256 = "5031ac14800798ae15cee79c04d65e326a575f2c968c7e2846a79bd07a1c0e61" +BLAKE2_FILES = { + "blake2b.blb": "00e098356d825dc75608e41a60c52b2e26ff06518dd84b262a781cb8b4a73d90", + "blake2s.blb": "c9de6782932db24c4510cea1b24ad67cd7f0834e974fae893957c12b1dea773e", +} + +BLAKE3_SOURCE_SHA256 = "dcb91ea8accc77e6d6e632af7cdc1a99a9f3ae78cf648da595c7d064db32f624" +BLAKE3_OUTPUT_SHA256 = "c56e08d48fc279088f99794e004bc774a76061a8705ed059a09dd9ea535e671d" + + +def fail(message: str) -> None: + raise SystemExit(f"hash-vector-provenance: {message}") + + +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def require_digest(path: Path, expected: str) -> bytes: + try: + data = path.read_bytes() + except OSError as error: + fail(f"cannot read {path}: {error}") + actual = digest(data) + if actual != expected: + fail(f"{path} has SHA-256 {actual}; expected {expected}") + return data + + +def require_git_commit(root: Path, expected: str) -> None: + try: + actual = subprocess.run( + ["git", "-C", str(root), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError) as error: + fail(f"cannot inspect upstream checkout {root}: {error}") + if actual != expected: + fail(f"{root} is at {actual}; expected {expected}") + + +def require_exact_artifacts(directory: Path, expected: set[str]) -> None: + actual = { + path.name + for path in directory.iterdir() + if path.is_file() and path.suffix in {".blb", ".json"} + } + if actual != expected: + missing = sorted(expected - actual) + extra = sorted(actual - expected) + fail(f"{directory} corpus coverage drift; missing={missing}, extra={extra}") + + +def encode_vlq(value: int) -> bytes: + encoded = [0, 0, 0, 0] + for index in (3, 2, 1, 0): + if index == 3: + encoded[index] = value & 0x7F + else: + value -= 1 + encoded[index] = 0x80 | (value & 0x7F) + value >>= 7 + if value == 0: + return bytes(encoded[index:]) + fail("vector blob is too large for the pinned VLQ format") + + +def encode_blobs_no_dedup(blobs: list[bytes]) -> bytes: + output = bytearray(b"\0") + for blob in blobs: + output.extend(encode_vlq(len(blob) << 1)) + output.extend(blob) + return bytes(output) + + +def verify_copied_family( + local_dir: Path, + source_dir: Path | None, + source_commit: str, + files: dict[str, str], +) -> None: + for name, expected in files.items(): + local = require_digest(local_dir / name, expected) + if source_dir is not None: + source = require_digest(source_dir / name, expected) + if source != local: + fail(f"{local_dir / name} differs from pinned upstream bytes") + if source_dir is not None: + require_git_commit(source_dir.parents[2], source_commit) + + +def blake2_outputs(source: bytes) -> dict[str, bytes]: + try: + cases = json.loads(source) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + fail(f"invalid BLAKE2 source JSON: {error}") + outputs: dict[str, bytes] = {} + for family in ("blake2b", "blake2s"): + selected = [case for case in cases if case.get("hash") == family] + if len(selected) != 512: + fail(f"BLAKE2 source has {len(selected)} {family} cases; expected 512") + blobs: list[bytes] = [] + try: + for case in selected: + blobs.extend(bytes.fromhex(case[field]) for field in ("in", "key", "out")) + except (KeyError, TypeError, ValueError) as error: + fail(f"invalid BLAKE2 {family} case: {error}") + outputs[f"{family}.blb"] = encode_blobs_no_dedup(blobs) + return outputs + + +def blake3_output(source: bytes) -> bytes: + try: + vectors = json.loads(source) + key = vectors["key"].encode() + context = vectors["context_string"].encode() + cases = vectors["cases"] + except (UnicodeDecodeError, json.JSONDecodeError, KeyError, AttributeError) as error: + fail(f"invalid BLAKE3 source JSON: {error}") + if len(cases) != 35: + fail(f"BLAKE3 source has {len(cases)} cases; expected 35") + blobs: list[bytes] = [] + try: + for case in cases: + blobs.extend( + ( + key, + context, + int(case["input_len"]).to_bytes(8, "little"), + bytes.fromhex(case["hash"]), + bytes.fromhex(case["keyed_hash"]), + bytes.fromhex(case["derive_key"]), + ) + ) + except (KeyError, TypeError, ValueError, OverflowError) as error: + fail(f"invalid BLAKE3 case: {error}") + return encode_blobs_no_dedup(blobs) + + +def verify_generated(output_dir: Path, generated: dict[str, bytes], expected: dict[str, str]) -> None: + for name, data in generated.items(): + if digest(data) != expected[name]: + fail(f"generated {name} has an unexpected digest") + if require_digest(output_dir / name, expected[name]) != data: + fail(f"{output_dir / name} differs from deterministic output") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--sha2-root", type=Path) + parser.add_argument("--sha3-root", type=Path) + parser.add_argument("--blake2-root", type=Path) + parser.add_argument("--blake3-root", type=Path) + args = parser.parse_args() + + require_exact_artifacts(ROOT / "testdata/sha2", set(SHA2_FILES)) + require_exact_artifacts(ROOT / "testdata/sha3", set(SHA3_FILES)) + require_exact_artifacts(ROOT / "testdata/blake2", set(BLAKE2_FILES)) + require_exact_artifacts( + ROOT / "testdata/blake3", + {"test_vectors.blb", "test_vectors.json"}, + ) + + verify_copied_family( + ROOT / "testdata/sha2", + args.sha2_root / "sha2/tests/data" if args.sha2_root else None, + SHA2_COMMIT, + SHA2_FILES, + ) + verify_copied_family( + ROOT / "testdata/sha3", + args.sha3_root / "sha3/tests/data" if args.sha3_root else None, + SHA3_COMMIT, + SHA3_FILES, + ) + + for name, expected in BLAKE2_FILES.items(): + require_digest(ROOT / "testdata/blake2" / name, expected) + if args.blake2_root: + require_git_commit(args.blake2_root, BLAKE2_COMMIT) + source = require_digest( + args.blake2_root / "testvectors/blake2-kat.json", + BLAKE2_SOURCE_SHA256, + ) + verify_generated(ROOT / "testdata/blake2", blake2_outputs(source), BLAKE2_FILES) + + local_blake3 = require_digest( + ROOT / "testdata/blake3/test_vectors.json", + BLAKE3_SOURCE_SHA256, + ) + generated_blake3 = blake3_output(local_blake3) + verify_generated( + ROOT / "testdata/blake3", + {"test_vectors.blb": generated_blake3}, + {"test_vectors.blb": BLAKE3_OUTPUT_SHA256}, + ) + if args.blake3_root: + require_git_commit(args.blake3_root, BLAKE3_COMMIT) + source = require_digest( + args.blake3_root / "test_vectors/test_vectors.json", + BLAKE3_SOURCE_SHA256, + ) + if source != local_blake3: + fail("committed BLAKE3 JSON differs from pinned upstream bytes") + + print("hash-vector-provenance: committed corpora match pinned digests and transforms") + + +if __name__ == "__main__": + main() diff --git a/scripts/check/rsa-asm-provenance.sh b/scripts/check/rsa-asm-provenance.sh new file mode 100755 index 00000000..65bd3051 --- /dev/null +++ b/scripts/check/rsa-asm-provenance.sh @@ -0,0 +1,755 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +MANIFEST="$REPO_ROOT/src/auth/asm/rscrypto_rsa_assembly_provenance.tsv" +ARCHIVE_PREFIX="aws-lc-sys-0.41.0/" +TRANSFORM="rsa-aws-lc-sys-0.41.0-transform-v1" +TRANSFORM_MANIFEST_SHA256="f0bc00139c7f44e793b5efb292a73650254a7335e739e95b6b46aff429bf8932" + +die() { + printf 'rsa-asm-provenance: %s\n' "$*" >&2 + exit 1 +} + +usage() { + printf 'usage: scripts/check/rsa-asm-provenance.sh [--archive PATH]\n' >&2 + exit 2 +} + +if command -v sha256sum >/dev/null 2>&1; then + hash_file() { + sha256sum "$1" | awk '{ print $1 }' + } +elif command -v shasum >/dev/null 2>&1; then + hash_file() { + shasum -a 256 "$1" | awk '{ print $1 }' + } +else + die "sha256sum or shasum is required" +fi + +require_shape() { + local path=$1 + local expected_lines=$2 + local expected_bytes=$3 + local expected_hash=$4 + local actual_lines actual_bytes actual_hash + + [[ -f "$path" ]] || die "missing file: $path" + actual_lines=$(wc -l <"$path" | tr -d ' ') + actual_bytes=$(wc -c <"$path" | tr -d ' ') + actual_hash=$(hash_file "$path") + [[ "$actual_lines" == "$expected_lines" ]] \ + || die "$path has $actual_lines lines; expected $expected_lines" + [[ "$actual_bytes" == "$expected_bytes" ]] \ + || die "$path has $actual_bytes bytes; expected $expected_bytes" + [[ "$actual_hash" == "$expected_hash" ]] \ + || die "$path has SHA-256 $actual_hash; expected $expected_hash" +} + +require_lf_file() { + local path=$1 + [[ -s "$path" ]] || die "empty transform input: $path" + if LC_ALL=C grep -q $'\r' "$path"; then + die "transform input is not LF-only: $path" + fi + [[ -z "$(tail -c 1 "$path")" ]] || die "transform input lacks a final LF: $path" +} + +validate_manifest() { + [[ -f "$MANIFEST" ]] || die "missing provenance manifest: $MANIFEST" + LC_ALL=C awk -F '\t' -v transform="$TRANSFORM" ' + BEGIN { + expected_member["aws-lc-sys-0.41.0/aws-lc/generated-src/ios-aarch64/crypto/fipsmodule/armv8-mont.S"] = 1 + expected_member["aws-lc-sys-0.41.0/aws-lc/generated-src/linux-x86_64/crypto/fipsmodule/x86_64-mont.S"] = 1 + expected_member["aws-lc-sys-0.41.0/aws-lc/generated-src/linux-x86_64/crypto/fipsmodule/x86_64-mont5.S"] = 1 + expected_member["aws-lc-sys-0.41.0/aws-lc/crypto/fipsmodule/bn/asm/armv8-mont.pl"] = 1 + expected_member["aws-lc-sys-0.41.0/aws-lc/crypto/fipsmodule/bn/asm/x86_64-mont.pl"] = 1 + expected_member["aws-lc-sys-0.41.0/aws-lc/crypto/fipsmodule/bn/asm/x86_64-mont5.pl"] = 1 + expected_member["aws-lc-sys-0.41.0/aws-lc/crypto/perlasm/arm-xlate.pl"] = 1 + expected_member["aws-lc-sys-0.41.0/aws-lc/crypto/perlasm/x86_64-xlate.pl"] = 1 + expected_output["src/auth/asm/rscrypto_rsa_bignum_mont_apple.s"] = 1 + expected_output["src/auth/asm/rscrypto_rsa_bignum_mont_aarch64_elf.s"] = 1 + expected_output["src/auth/asm/rscrypto_rsa_x86_64_elf.S"] = 1 + } + /^#/ { next } + $1 == "schema" { + if (NF != 2 || $2 != "1") exit 2 + schema++ + next + } + $1 == "transform" { + if (NF != 2 || $2 != transform) exit 2 + transforms++ + next + } + $1 == "archive" { + if (NF != 4 || $2 != "aws-lc-sys-0.41.0" || + length($3) != 64 || $3 !~ /^[0-9a-f]+$/ || + length($4) != 40 || $4 !~ /^[0-9a-f]+$/) exit 2 + archives++ + next + } + $1 == "member" { + if (NF != 3 || index($2, "aws-lc-sys-0.41.0/aws-lc/") != 1 || + length($3) != 64 || $3 !~ /^[0-9a-f]+$/ || + !($2 in expected_member) || seen_member[$2]++) exit 2 + members++ + next + } + $1 == "output" { + if (NF != 5 || index($2, "src/auth/asm/rscrypto_rsa_") != 1 || + $3 !~ /^[1-9][0-9]*$/ || $4 !~ /^[1-9][0-9]*$/ || + length($5) != 64 || $5 !~ /^[0-9a-f]+$/ || + !($2 in expected_output) || seen_output[$2]++) exit 2 + outputs++ + next + } + { exit 2 } + END { + if (schema != 1 || transforms != 1 || archives != 1 || members != 8 || outputs != 3) exit 2 + for (path in expected_member) if (!(path in seen_member)) exit 2 + for (path in expected_output) if (!(path in seen_output)) exit 2 + } + ' "$MANIFEST" || die "invalid provenance manifest" + [[ "$(hash_file "$MANIFEST")" == "$TRANSFORM_MANIFEST_SHA256" ]] \ + || die "provenance manifest changed without a new transform identity" +} + +verify_external_rsa_coverage() { + local manifest_outputs external_outputs + + manifest_outputs="$( + awk -F '\t' '$1 == "output" { print $2 }' "$MANIFEST" | + LC_ALL=C sort + )" + external_outputs="$( + find "$REPO_ROOT/src" -type f \ + \( -name '*.s' -o -name '*.S' \) | + while IFS= read -r path; do + relative=${path#"$REPO_ROOT/"} + if grep -Eq \ + '^(//|[[:space:]]*\*) Adapted for rscrypto|^[[:space:]]*\* The butterfly schedule is auto-derived from' \ + "$path" && + { + grep -Eq '(^|/)rsa([/_.]|$)|_rsa([_.]|$)' <<<"$relative" || + grep -Eq \ + '^[[:space:]]*\.(globl|global)[[:space:]]+_?rscrypto_rsa_[A-Za-z0-9_]+([[:space:]]|$)|^_?rscrypto_rsa_[A-Za-z0-9_]+:' \ + "$path" + }; then + printf '%s\n' "$relative" + fi + done | + LC_ALL=C sort + )" + [[ "$external_outputs" == "$manifest_outputs" ]] \ + || die "external-derived RSA assembly does not exactly match manifest outputs" +} + +verify_rsa_wrapper_fingerprints() { + local wrapper expected actual + + while IFS=$'\t' read -r wrapper expected; do + actual=$(hash_file "$REPO_ROOT/$wrapper") + [[ "$actual" == "$expected" ]] \ + || die "$wrapper changed without a provenance review" + done <<'EOF' +src/auth/rsa_aarch64_asm.rs 65c059365bb76d948d2f09873870ba7e146af15e52e10560bf632d39bdd33834 +src/auth/rsa_aarch64_linux_asm.rs 2b311eb4f253ff458492c76f52f35c0a4d5ae8705774e568ae853e1f6ea1a049 +src/auth/rsa_x86_64_asm.rs 98bc241435de70963c23f0e98431024585f95eed268b773a42d3bd2983b8038e +EOF +} + +verify_rsa_wrapper_coverage() { + local manifest_outputs wrapper_targets wrapper include path + + manifest_outputs="$( + awk -F '\t' '$1 == "output" { print $2 }' "$MANIFEST" | + LC_ALL=C sort + )" + wrapper_targets="$( + for wrapper in \ + "$REPO_ROOT/src/auth/rsa_aarch64_asm.rs" \ + "$REPO_ROOT/src/auth/rsa_aarch64_linux_asm.rs" \ + "$REPO_ROOT/src/auth/rsa_x86_64_asm.rs"; do + [[ -f "$wrapper" ]] || die "missing RSA assembly wrapper: $wrapper" + while IFS= read -r include; do + [[ "$include" != /* && "$include" != *..* ]] \ + || die "unsupported RSA assembly include path: $include" + path="$REPO_ROOT/src/auth/$include" + [[ -f "$path" ]] || die "missing RSA assembly include target: $path" + if grep -Eq \ + '^(//|[[:space:]]*\*) Adapted for rscrypto|^[[:space:]]*\* The butterfly schedule is auto-derived from' \ + "$path"; then + printf 'src/auth/%s\n' "$include" + fi + done < <( + awk ' + { + line = $0 + while (match(line, /include_str!\("[^"]+"\)/)) { + token = substr(line, RSTART, RLENGTH) + sub(/^include_str!\("/, "", token) + sub(/"\)$/, "", token) + print token + line = substr(line, RSTART + RLENGTH) + } + } + ' "$wrapper" + ) + done | + LC_ALL=C sort + )" + [[ "$wrapper_targets" == "$manifest_outputs" ]] \ + || die "external RSA assembly wrapper targets do not exactly match manifest outputs" +} + +verify_committed_outputs() { + while IFS=$'\t' read -r kind path lines bytes digest; do + [[ "$kind" == "output" ]] || continue + case "$path" in + src/auth/asm/rscrypto_rsa_bignum_mont_apple.s \ + | src/auth/asm/rscrypto_rsa_bignum_mont_aarch64_elf.s \ + | src/auth/asm/rscrypto_rsa_x86_64_elf.S) ;; + *) die "unexpected RSA provenance output: $path" ;; + esac + require_shape "$REPO_ROOT/$path" "$lines" "$bytes" "$digest" + done <"$MANIFEST" +} + +validate_manifest +verify_external_rsa_coverage +verify_rsa_wrapper_fingerprints +verify_rsa_wrapper_coverage +verify_committed_outputs + +if [[ $# -eq 0 ]]; then + printf 'rsa-asm-provenance: committed RSA assembly matches the provenance manifest\n' + exit 0 +fi +[[ $# -eq 2 && $1 == "--archive" ]] || usage +ARCHIVE=$2 +[[ -f "$ARCHIVE" ]] || die "archive is not a regular file: $ARCHIVE" +export LC_ALL=C + +TEMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/rscrypto-rsa-asm.XXXXXXXX")" +[[ -d "$TEMP_DIR" ]] || die "failed to create temporary directory" +trap 'rm -rf -- "$TEMP_DIR"' EXIT + +archive_digest=$(awk -F '\t' '$1 == "archive" { print $3 }' "$MANIFEST") +actual_archive_digest=$(hash_file "$ARCHIVE") +[[ "$actual_archive_digest" == "$archive_digest" ]] \ + || die "archive SHA-256 $actual_archive_digest does not match $archive_digest" + +ARCHIVE_LIST="$TEMP_DIR/archive.list" +tar -tzf "$ARCHIVE" >"$ARCHIVE_LIST" + +members=() +while IFS=$'\t' read -r kind member digest; do + [[ "$kind" == "member" ]] || continue + members+=("$member") + exact_count=$(awk -v member="$member" '$0 == member { count++ } END { print count + 0 }' "$ARCHIVE_LIST") + [[ "$exact_count" == 1 ]] || die "archive must contain exactly one $member" + relative=${member#"$ARCHIVE_PREFIX"} + suffix_count=$(awk -v suffix="/$relative" ' + length($0) >= length(suffix) && substr($0, length($0) - length(suffix) + 1) == suffix { count++ } + END { print count + 0 } + ' "$ARCHIVE_LIST") + [[ "$suffix_count" == 1 ]] || die "archive contains required member under another prefix: $relative" +done <"$MANIFEST" +[[ ${#members[@]} -eq 8 ]] || die "manifest member count changed during extraction" + +tar -xzf "$ARCHIVE" -C "$TEMP_DIR" "${members[@]}" +while IFS=$'\t' read -r kind member digest; do + [[ "$kind" == "member" ]] || continue + require_shape "$TEMP_DIR/$member" "$(wc -l <"$TEMP_DIR/$member" | tr -d ' ')" \ + "$(wc -c <"$TEMP_DIR/$member" | tr -d ' ')" "$digest" +done <"$MANIFEST" + +SOURCE_ROOT="$TEMP_DIR/$ARCHIVE_PREFIX/aws-lc" +APPLE_INPUT="$SOURCE_ROOT/generated-src/ios-aarch64/crypto/fipsmodule/armv8-mont.S" +X86_MONT_INPUT="$SOURCE_ROOT/generated-src/linux-x86_64/crypto/fipsmodule/x86_64-mont.S" +X86_MONT5_INPUT="$SOURCE_ROOT/generated-src/linux-x86_64/crypto/fipsmodule/x86_64-mont5.S" +require_lf_file "$APPLE_INPUT" +require_lf_file "$X86_MONT_INPUT" +require_lf_file "$X86_MONT5_INPUT" + +APPLE_CLOSE="#endif // !OPENSSL_NO_ASM && defined(OPENSSL_AARCH64) && defined(__APPLE__)" +APPLE_OUTER="#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) && defined(__APPLE__)" +APPLE_SIGNATURE=$'.byte\t77,111,110,116,103,111,109,101,114,121,32,77,117,108,116,105,112,108,105,99,97,116,105,111,110,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0' + +awk -v closing="$APPLE_CLOSE" -v outer="$APPLE_OUTER" -v signature="$APPLE_SIGNATURE" ' + function is_ident(c) { return c ~ /^[A-Za-z0-9_]$/ } + function token_count(s, token, count, pos, before, after) { + count = 0 + while ((pos = index(s, token)) != 0) { + before = pos == 1 ? "" : substr(s, pos - 1, 1) + after = substr(s, pos + length(token), 1) + if ((before == "" || !is_ident(before)) && (after == "" || !is_ident(after))) count++ + s = substr(s, pos + length(token)) + } + return count + } + $0 == outer { outer_count++; outer_seen = 1; next } + $0 == ".text" { + starts++ + if (!outer_seen || inside) exit 2 + inside = 1 + } + $0 == closing { + closes++ + if (!inside) exit 2 + inside = 0 + next + } + inside { + lines++ + if ($0 ~ /^[[:space:]]*[.]cfi_/) cfi++ + if ($0 == "\tAARCH64_SIGN_LINK_REGISTER") sign++ + if ($0 == "\tAARCH64_VALIDATE_LINK_REGISTER") validate++ + if ($0 == "\t// Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_sqr8x_mont is jumped to" || + $0 == "\t// Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_mul4x_mont is jumped to") pac_comments++ + if (index($0, "AARCH64_SIGN_LINK_REGISTER") || index($0, "AARCH64_VALIDATE_LINK_REGISTER")) pac_text++ + if ($0 == "\tmov\tx0,#1") returns++ + bn_mul += token_count($0, "_bn_mul_mont") + sqr += token_count($0, "__bn_sqr8x_mont") + mul4 += token_count($0, "__bn_mul4x_mont") + rewinded += gsub(/rewinded/, "&") + only_which += gsub(/only from bn_mul_mont which/, "&") + only_or += gsub(/only from bn_mul_mont or/, "&") + if ($0 == "\tmov\tsp,x22\t\t\t// alloca") stack8++ + if ($0 == "\tsub\tx2,sp,x5,lsl#4") stack16++ + if ($0 == "\tsub\tx26,sp,x5,lsl#3") stack8b++ + last3 = last2 + last2 = last1 + last1 = $0 + } + END { + if (outer_count != 1 || starts != 1 || closes != 1 || inside || + lines != 1495 || cfi != 79 || sign != 1 || validate != 3 || + pac_comments != 2 || pac_text != 6 || returns != 3 || + bn_mul != 3 || sqr != 3 || mul4 != 4 || rewinded != 4 || + only_which != 1 || only_or != 1 || stack8 != 1 || stack16 != 1 || stack8b != 1 || + last3 != signature || last2 != ".align\t2" || last1 != ".align\t4") exit 2 + } +' "$APPLE_INPUT" || die "Apple AArch64 transform preflight failed" + +APPLE_BODY="$TEMP_DIR/apple.body" +awk -v closing="$APPLE_CLOSE" ' + function is_ident(c) { return c ~ /^[A-Za-z0-9_]$/ } + function replace_token(s, old, new, out, pos, before, after) { + out = "" + while ((pos = index(s, old)) != 0) { + before = pos == 1 ? "" : substr(s, pos - 1, 1) + after = substr(s, pos + length(old), 1) + if ((before == "" || !is_ident(before)) && (after == "" || !is_ident(after))) { + out = out substr(s, 1, pos - 1) new + s = substr(s, pos + length(old)) + replacements++ + } else { + out = out substr(s, 1, pos) + s = substr(s, pos + 1) + } + } + return out s + } + $0 == ".text" { inside = 1 } + $0 == closing { inside = 0; next } + !inside { next } + /^[[:space:]]*[.]cfi_/ { cfi++; next } + $0 == "\tAARCH64_SIGN_LINK_REGISTER" || $0 == "\tAARCH64_VALIDATE_LINK_REGISTER" { pac++; next } + $0 == "\t// Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_sqr8x_mont is jumped to" || + $0 == "\t// Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_mul4x_mont is jumped to" { + pac_comments++ + next + } + { + line = $0 + if (line == "\tmov\tx0,#1") { + line = "\t// No return value" + returns++ + } + which += gsub(/only from bn_mul_mont which/, "only from bn_mul_mont_words which", line) + only_or += gsub(/only from bn_mul_mont or/, "only from bn_mul_mont_words or", line) + rewinded += gsub(/rewinded/, "rewound", line) + before = replacements + line = replace_token(line, "_bn_mul_mont", "_rscrypto_rsa_bn_mul_mont_words_apple") + bn_mul += replacements - before + before = replacements + line = replace_token(line, "__bn_sqr8x_mont", "Lrscrypto_rsa_bn_sqr8x_mont") + sqr += replacements - before + before = replacements + line = replace_token(line, "__bn_mul4x_mont", "Lrscrypto_rsa_bn_mul4x_mont") + mul4 += replacements - before + if (line == "\tmov\tsp,x22\t\t\t// alloca") { + print "\t// This can allocate at most 8 * BN_MONTGOMERY_MAX_WORDS on the stack," + print "\t// or 2 KiB. This fits well within a page, so it is not necessary to" + print "\t// fault pages in the correct order." + insert8++ + } + if (line == "\tsub\tx2,sp,x5,lsl#4") { + print "\t// This can allocate at most 16 * BN_MONTGOMERY_MAX_WORDS on the stack," + print "\t// or 4 KiB. The fixed allocation above pushes to just above a page. On" + print "\t// Windows, we must ensure new pages are first accessed in order. See" + print "\t// https://learn.microsoft.com/en-us/cpp/build/arm64-windows-abi-conventions?view=msvc-170#stack" + print "\t//" + print "\t// The order is correct, but precariously so: the code above access as" + print "\t// low as [sp,#16]. This leaves a jump of 16 + 4096 = 4112 bytes. If" + print "\t// [sp,#16] were at page boundary, those 4112 bytes would span two" + print "\t// pages. If [x2] were the next access, we would skip a guard page." + print "\t//" + print "\t// Fortunately, the first access is [x2,#8*8], at .Lsqr8x_zero_start." + print "\t// We jump at most 4112 - 64 = 4048 bytes, less than a page. If any of" + print "\t// this changes, we must insert a no-op access or call __chkstk." + insert16++ + } + if (line == "\tsub\tx26,sp,x5,lsl#3") { + print "\t// This can allocate at most 8 * BN_MONTGOMERY_MAX_WORDS on the stack," + print "\t// or 2 KiB. This fits well within a page, so it is not necessary to" + print "\t// fault pages in the correct order." + insert8b++ + } + print line + } + END { + if (inside || cfi != 79 || pac != 4 || pac_comments != 2 || returns != 3 || + which != 1 || only_or != 1 || rewinded != 4 || bn_mul != 3 || sqr != 2 || mul4 != 3 || + insert8 != 1 || insert16 != 1 || insert8b != 1) exit 2 + } +' "$APPLE_INPUT" >"$APPLE_BODY" || die "Apple AArch64 transform failed" +require_shape "$APPLE_BODY" 1429 31667 1238c64546b882e5763689d6e8c7bb38999341cc2bfd8a344a4f044722cb9669 + +APPLE_OUTPUT="$TEMP_DIR/rscrypto_rsa_bignum_mont_apple.s" +{ + printf '%s\n' \ + '// Copyright 2015-2016 The OpenSSL Project Authors. All Rights Reserved.' \ + '// SPDX-License-Identifier: Apache-2.0' \ + '//' \ + '// Adapted for rscrypto from BoringSSL generated armv8-mont-apple.S.' \ + '// The public symbol is renamed into the rscrypto namespace and embedded with Rust global_asm!.' \ + '' \ + '// This file is generated from a similarly-named Perl script in the BoringSSL' \ + '// source tree. Do not edit by hand.' \ + '' \ + '' + awk '{ print }' "$APPLE_BODY" +} >"$APPLE_OUTPUT" +require_shape "$APPLE_OUTPUT" 1439 32065 3e723bd775c6e216d9525d470d6d082a83e989f7988db0355c0ad9cd5bfa5072 +cmp -s "$APPLE_OUTPUT" "$REPO_ROOT/src/auth/asm/rscrypto_rsa_bignum_mont_apple.s" \ + || die "reconstructed Apple assembly differs from the committed snapshot" + +AARCH64_ELF_OUTPUT="$TEMP_DIR/rscrypto_rsa_bignum_mont_aarch64_elf.s" +awk ' + function is_ident(c) { return c ~ /^[A-Za-z0-9_]$/ } + function replace_token(s, old, new, out, pos, before, after) { + out = "" + while ((pos = index(s, old)) != 0) { + before = pos == 1 ? "" : substr(s, pos - 1, 1) + after = substr(s, pos + length(old), 1) + if ((before == "" || !is_ident(before)) && (after == "" || !is_ident(after))) { + out = out substr(s, 1, pos - 1) new + s = substr(s, pos + length(old)) + replacements++ + } else { + out = out substr(s, 1, pos) + s = substr(s, pos + 1) + } + } + return out s + } + { + line = $0 + if (line == "// Adapted for rscrypto from BoringSSL generated armv8-mont-apple.S.") { + line = "// Adapted for rscrypto from BoringSSL generated armv8-mont-apple.S and retargeted for ELF." + header++ + } + before = replacements + line = replace_token(line, "_rscrypto_rsa_bn_mul_mont_words_apple", "rscrypto_rsa_bn_mul_mont_words_aarch64_elf") + symbol += replacements - before + before = replacements + line = replace_token(line, ".private_extern", ".hidden") + directive += replacements - before + print line + } + END { + if (header != 1 || symbol != 3 || directive != 1) exit 2 + } +' "$APPLE_OUTPUT" >"$AARCH64_ELF_OUTPUT" || die "AArch64 ELF retarget transform failed" +require_shape "$AARCH64_ELF_OUTPUT" 1439 32095 f28fa8f4f02e0427288c717fc5666b7c07f0f8fc0c7ea5bb21721b1ffe3dc4dc +cmp -s "$AARCH64_ELF_OUTPUT" "$REPO_ROOT/src/auth/asm/rscrypto_rsa_bignum_mont_aarch64_elf.s" \ + || die "reconstructed AArch64 ELF assembly differs from the committed snapshot" + +X86_OUTER="#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__ELF__)" +X86_SIGNATURE=$'.byte\t77,111,110,116,103,111,109,101,114,121,32,77,117,108,116,105,112,108,105,99,97,116,105,111,110,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0' +awk -v outer="$X86_OUTER" -v signature="$X86_SIGNATURE" ' + function is_ident(c) { return c ~ /^[A-Za-z0-9_]$/ } + function token_count(s, token, count, pos, before, after) { + count = 0 + while ((pos = index(s, token)) != 0) { + before = pos == 1 ? "" : substr(s, pos - 1, 1) + after = substr(s, pos + length(token), 1) + if ((before == "" || !is_ident(before)) && (after == "" || !is_ident(after))) count++ + s = substr(s, pos + length(token)) + } + return count + } + $0 == outer { outer_count++; outer_seen = 1; next } + $0 == ".text\t" { + starts++ + if (!outer_seen || inside) exit 2 + inside = 1 + } + inside { + lines++ + if ($0 == "_CET_ENDBR") cet++ + if ($0 == "#ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX") { + if (guard_depth != 0) exit 2 + guard_depth = 1 + guards++ + } + if ($0 == "#endif") { + if (guard_depth != 1) exit 2 + guard_depth = 0 + guard_ends++ + } + nohw += token_count($0, "bn_mul_mont_nohw") + mul4 += token_count($0, "bn_mul4x_mont") + sqr8 += token_count($0, "bn_sqr8x_mont") + mulx4 += token_count($0, "bn_mulx4x_mont") + sqrx += token_count($0, "bn_sqrx8x_internal") + sqr += token_count($0, "bn_sqr8x_internal") + if ($0 == signature) { + signatures++ + after_signature = 1 + } else if (after_signature && $0 == ".align\t16") { + if (guard_depth != 0) exit 2 + terminals++ + inside = 0 + expect_close = 1 + } + next + } + expect_close { + if ($0 != "#endif") exit 2 + outer_close++ + expect_close = 0 + } + END { + if (outer_count != 1 || starts != 1 || signatures != 1 || terminals != 1 || + outer_close != 1 || inside || expect_close || lines != 1236 || + guard_depth != 0 || + cet != 4 || guards != 3 || guard_ends != 3 || + nohw != 6 || mul4 != 6 || sqr8 != 6 || mulx4 != 6 || sqrx != 3 || sqr != 3) exit 2 + } +' "$X86_MONT_INPUT" || die "x86-64 mont transform preflight failed" + +X86_MONT_BODY="$TEMP_DIR/x86-mont.body" +awk -v signature="$X86_SIGNATURE" ' + function is_ident(c) { return c ~ /^[A-Za-z0-9_]$/ } + function replace_token(s, old, new, out, pos, before, after) { + out = "" + while ((pos = index(s, old)) != 0) { + before = pos == 1 ? "" : substr(s, pos - 1, 1) + after = substr(s, pos + length(old), 1) + if ((before == "" || !is_ident(before)) && (after == "" || !is_ident(after))) { + out = out substr(s, 1, pos - 1) new + s = substr(s, pos + length(old)) + replacements++ + } else { + out = out substr(s, 1, pos) + s = substr(s, pos + 1) + } + } + return out s + } + $0 == ".text\t" { inside = 1 } + !inside { next } + $0 == "_CET_ENDBR" { cet++; next } + $0 == "#ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX" { + if (guard_depth != 0) exit 2 + guard_depth = 1 + guards++ + next + } + $0 == "#endif" { + if (guard_depth != 1) exit 2 + guard_depth = 0 + guards++ + next + } + { + line = $0 + before = replacements + line = replace_token(line, "bn_mul_mont_nohw", "rscrypto_rsa_bn_mul_mont_nohw_x86_64_elf") + nohw += replacements - before + before = replacements + line = replace_token(line, "bn_mul4x_mont", "rscrypto_rsa_bn_mul4x_mont_x86_64_elf") + mul4 += replacements - before + before = replacements + line = replace_token(line, "bn_sqr8x_mont", "rscrypto_rsa_bn_sqr8x_mont_x86_64_elf") + sqr8 += replacements - before + before = replacements + line = replace_token(line, "bn_mulx4x_mont", "rscrypto_rsa_bn_mulx4x_mont_x86_64_elf") + mulx4 += replacements - before + before = replacements + line = replace_token(line, "bn_sqrx8x_internal", "rscrypto_rsa_bn_sqrx8x_internal_x86_64_elf") + sqrx += replacements - before + before = replacements + line = replace_token(line, "bn_sqr8x_internal", "rscrypto_rsa_bn_sqr8x_internal_x86_64_elf") + sqr += replacements - before + print line + if ($0 == signature) after_signature = 1 + else if (after_signature && $0 == ".align\t16") { + if (guard_depth != 0) exit 2 + inside = 0 + } + } + END { + if (inside || guard_depth != 0 || cet != 4 || guards != 6 || nohw != 6 || mul4 != 6 || + sqr8 != 6 || mulx4 != 6 || sqrx != 3 || sqr != 3) exit 2 + } +' "$X86_MONT_INPUT" >"$X86_MONT_BODY" || die "x86-64 mont transform failed" +require_shape "$X86_MONT_BODY" 1226 20955 2e9a43f8690820be1c17aac0b68c6ae9092e09bc28775460d6b93334e4f7dd96 + +X86_MONT5_BODY="$TEMP_DIR/x86-mont5.body" +awk ' + function is_ident(c) { return c ~ /^[A-Za-z0-9_]$/ } + function replace_token(s, old, new, out, pos, before, after) { + out = "" + while ((pos = index(s, old)) != 0) { + before = pos == 1 ? "" : substr(s, pos - 1, 1) + after = substr(s, pos + length(old), 1) + if ((before == "" || !is_ident(before)) && (after == "" || !is_ident(after))) { + out = out substr(s, 1, pos - 1) new + s = substr(s, pos + length(old)) + replacements++ + } else { + out = out substr(s, 1, pos) + s = substr(s, pos + 1) + } + } + return out s + } + $0 == "#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__ELF__)" { openings++; outer_depth++ } + $0 == "#ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX" { openings++; my_depth++ } + $0 == "#endif" { + closings++ + if (my_depth > 0) my_depth-- + else if (outer_depth > 0) outer_depth-- + } + $0 == ".globl\tbn_sqr8x_internal" { + start1++ + if (my_depth != 0 || block) exit 2 + block = 1 + } + $0 == ".globl\tbn_sqrx8x_internal" { + start2++ + if (my_depth != 1 || block) exit 2 + block = 2 + } + block { + raw_lines++ + if (expect_alias) { + expected_alias = block == 1 ? "__bn_sqr8x_internal:" : "__bn_sqrx8x_internal:" + if ($0 != expected_alias) exit 2 + aliases++ + expect_alias = 0 + expect_cfi = 1 + next + } + if (expect_cfi) { + if ($0 != ".cfi_startproc\t") exit 2 + expect_cfi = 0 + expect_cet = 1 + } else if (expect_cet) { + if ($0 != "_CET_ENDBR") exit 2 + expect_cet = 0 + cet++ + skip_blanks = 1 + blank_count = 0 + next + } + if (skip_blanks) { + if ($0 == "") { + blank_count++ + next + } + expected_blanks = cet == 1 ? 73 : 40 + expected_anchor = cet == 1 ? "\tleaq\t32(%r10),%rbp" : "\tleaq\t48+8(%rsp),%rdi" + if (blank_count != expected_blanks || $0 != expected_anchor) exit 2 + skip_blanks = 0 + blank_runs++ + } + if ($0 == "__bn_sqr8x_internal:" || $0 == "__bn_sqrx8x_internal:") { + exit 2 + } + if ($0 == "_CET_ENDBR") { + exit 2 + } + if ($0 == "bn_sqr8x_internal:" || $0 == "bn_sqrx8x_internal:") { + expected_public = block == 1 ? "bn_sqr8x_internal:" : "bn_sqrx8x_internal:" + if ($0 != expected_public || public_labels == block) exit 2 + public_labels = block + expect_alias = 1 + } + line = $0 + before = replacements + line = replace_token(line, "bn_sqr8x_internal", "rscrypto_rsa_bn_sqr8x_internal_x86_64_elf") + sqr += replacements - before + before = replacements + line = replace_token(line, "bn_sqrx8x_internal", "rscrypto_rsa_bn_sqrx8x_internal_x86_64_elf") + sqrx += replacements - before + before = replacements + line = replace_token(line, "__bn_sqr8x_reduction", "rscrypto_rsa_bn_sqr8x_reduction_x86_64_elf") + sqr_red += replacements - before + before = replacements + line = replace_token(line, "__bn_sqrx8x_reduction", "rscrypto_rsa_bn_sqrx8x_reduction_x86_64_elf") + sqrx_red += replacements - before + print line + if ($0 == ".size\tbn_sqr8x_internal,.-bn_sqr8x_internal") { + end1++ + block = 0 + } else if ($0 == ".size\tbn_sqrx8x_internal,.-bn_sqrx8x_internal") { + end2++ + block = 0 + } + } + END { + if (openings != 5 || closings != 5 || outer_depth || my_depth || block || + start1 != 1 || end1 != 1 || start2 != 1 || end2 != 1 || raw_lines != 1407 || + public_labels != 2 || aliases != 2 || cet != 2 || blank_runs != 2 || skip_blanks || + expect_alias || expect_cfi || expect_cet || + sqr != 7 || sqrx != 7 || sqr_red != 1 || sqrx_red != 1) exit 2 + } +' "$X86_MONT5_INPUT" >"$X86_MONT5_BODY" || die "x86-64 mont5 transform failed" +require_shape "$X86_MONT5_BODY" 1290 21372 5baa0aa7f587bee6049d66e1d148b31f73ff6a4cd5a4864310770d7c854d9552 + +X86_OUTPUT="$TEMP_DIR/rscrypto_rsa_x86_64_elf.S" +{ + printf '%s\n' \ + '// Copyright 2005-2016 The OpenSSL Project Authors. All Rights Reserved.' \ + '// SPDX-License-Identifier: Apache-2.0' \ + '//' \ + '// Adapted for rscrypto from AWS-LC/BoringSSL generated x86_64-mont.S and x86_64-mont5.S.' \ + '// Public symbols are renamed into the rscrypto namespace and embedded with Rust global_asm!.' \ + '' \ + '// This file is generated from a similarly-named Perl script in the BoringSSL' \ + '// source tree. Do not edit by hand.' \ + '' \ + '' + awk '{ print }' "$X86_MONT_BODY" + printf '\n%s\n' '// Square internals required by rscrypto_rsa_bn_sqr8x_mont_x86_64_elf.' + awk '{ print }' "$X86_MONT5_BODY" + printf '\n%s\n' '.section .note.GNU-stack,"",@progbits' +} >"$X86_OUTPUT" + +if grep -Eq '(^|[^A-Za-z0-9_])(bn_mul_mont_nohw|bn_mul4x_mont|bn_sqr8x_mont|bn_mulx4x_mont|bn_sqrx8x_internal|bn_sqr8x_internal)([^A-Za-z0-9_]|$)|^_CET_ENDBR$|^#(if|endif)' "$X86_OUTPUT"; then + die "x86-64 postflight found an old token, CET marker, or preprocessor guard" +fi +require_shape "$X86_OUTPUT" 2530 42855 1b59e82724d10b06decbc8a9fbc88f9e04cefdee0524ef752c37ccf422f729a3 +cmp -s "$X86_OUTPUT" "$REPO_ROOT/src/auth/asm/rscrypto_rsa_x86_64_elf.S" \ + || die "reconstructed x86-64 assembly differs from the committed snapshot" + +printf 'rsa-asm-provenance: archive, members, transforms, and committed snapshots verified\n' diff --git a/scripts/check/signature-asm-provenance.py b/scripts/check/signature-asm-provenance.py new file mode 100755 index 00000000..b78ea91c --- /dev/null +++ b/scripts/check/signature-asm-provenance.py @@ -0,0 +1,641 @@ +#!/usr/bin/env python3 +"""Verify non-RSA signature assembly provenance and deterministic transforms.""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import re +import subprocess +import tempfile +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Callable + +ROOT = Path(__file__).resolve().parents[2] +MANIFEST = ROOT / "src/auth/asm/rscrypto_signature_assembly_provenance.tsv" +UPSTREAM_URL = "https://github.com/awslabs/s2n-bignum.git" +MANIFEST_SHA256 = "2cec9cf09bac661e7235a6e50d0e5c97e0a38d7c8e32e03e908a34088378ad5a" + +PINNED_COMMIT = "471fca76a9079753aab938ba35ef55ec22717d89" +ED25519_X86_COMMIT = "c19516a30de81f9e664dccdfc79dbf8fb109276d" +X25519_X86_COMMIT = "333cdfcd91a62d15954ecca1124544b8587f86de" + +SHA256_RE = re.compile(r"[0-9a-f]{64}") +COMMIT_RE = re.compile(r"[0-9a-f]{40}") +BODY_RE = re.compile(br"(?m)^[ \t]*\.globl[ \t]+") +GLOBAL_RE = re.compile(br"(?m)^[ \t]*\.globl[ \t]+([.$A-Za-z_][.$A-Za-z0-9_]*)[ \t]*$") +SOURCE_SYMBOL_RE = re.compile( + br"S2N_BN_(?:SYMBOL|SYM_VISIBILITY_DIRECTIVE)\(([A-Za-z_][A-Za-z0-9_]*)\)" +) +SYMBOL_BYTE = rb"A-Za-z0-9_.$" + + +@dataclass(frozen=True) +class Source: + member: str + sha256: str + + +@dataclass(frozen=True) +class Entry: + local_path: str + local_sha256: str + generated_sha256: str + commit: str + transform: str + sources: tuple[Source, ...] + + +def fail(message: str) -> None: + raise SystemExit(f"signature-asm-provenance: {message}") + + +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def read_file(path: Path) -> bytes: + try: + return path.read_bytes() + except OSError as error: + fail(f"cannot read {path}: {error}") + + +def safe_relative_path(value: str, prefix: str | None = None) -> None: + path = PurePosixPath(value) + if path.is_absolute() or ".." in path.parts or str(path) != value: + fail(f"unsafe or non-canonical manifest path: {value}") + if prefix is not None and not value.startswith(prefix): + fail(f"manifest path {value} is outside {prefix}") + + +def parse_sources(value: str, line_number: int) -> tuple[Source, ...]: + sources: list[Source] = [] + seen: set[str] = set() + for field in value.split(","): + if field.count("=") != 1: + fail(f"manifest line {line_number} has an invalid source field") + member, sha256 = field.split("=", 1) + safe_relative_path(member) + if not member.startswith(("arm/", "x86_att/")) or not member.endswith(".S"): + fail(f"manifest line {line_number} has unsupported source member {member}") + if not SHA256_RE.fullmatch(sha256): + fail(f"manifest line {line_number} has invalid source SHA-256") + if member in seen: + fail(f"manifest line {line_number} repeats source member {member}") + seen.add(member) + sources.append(Source(member, sha256)) + if not sources: + fail(f"manifest line {line_number} has no source members") + return tuple(sources) + + +def parse_manifest() -> list[Entry]: + data = read_file(MANIFEST) + if digest(data) != MANIFEST_SHA256: + fail("provenance manifest changed without a transform review") + try: + lines = data.decode("utf-8").splitlines() + except UnicodeDecodeError as error: + fail(f"manifest is not UTF-8: {error}") + records = [ + (line_number, line.split("\t")) + for line_number, line in enumerate(lines, 1) + if line and not line.startswith("#") + ] + if len(records) != 38: + fail(f"manifest has {len(records)} records; expected 38") + if records[0][1] != ["schema", "1"]: + fail("manifest must begin with schema version 1") + if records[1][1] != ["upstream", UPSTREAM_URL]: + fail(f"manifest must bind upstream {UPSTREAM_URL}") + + entries: list[Entry] = [] + for line_number, fields in records[2:]: + if len(fields) != 7 or fields[0] != "output": + fail(f"manifest line {line_number} is not a seven-field output record") + _, local_path, local_sha256, generated_sha256, commit, transform, source_field = fields + safe_relative_path(local_path, "src/auth/") + if Path(local_path).suffix not in {".s", ".S"}: + fail(f"manifest line {line_number} does not name an assembly snapshot") + if not SHA256_RE.fullmatch(local_sha256): + fail(f"manifest line {line_number} has invalid local SHA-256") + if not SHA256_RE.fullmatch(generated_sha256): + fail(f"manifest line {line_number} has invalid generated SHA-256") + if not COMMIT_RE.fullmatch(commit): + fail(f"manifest line {line_number} has invalid upstream commit") + entries.append( + Entry( + local_path, + local_sha256, + generated_sha256, + commit, + transform, + parse_sources(source_field, line_number), + ) + ) + + paths = [entry.local_path for entry in entries] + if len(entries) != 36: + fail(f"manifest has {len(entries)} outputs; expected 36") + if len(set(paths)) != len(paths): + fail("manifest repeats a local output") + if paths != sorted(paths): + fail("manifest outputs are not in bytewise path order") + + expected_transforms = set(TRANSFORMS) + actual_transforms = {entry.transform for entry in entries} + if actual_transforms != expected_transforms: + fail( + "manifest transform set differs from verifier transforms: " + f"{sorted(actual_transforms ^ expected_transforms)}" + ) + allowed_commits = {PINNED_COMMIT, ED25519_X86_COMMIT, X25519_X86_COMMIT} + if {entry.commit for entry in entries} - allowed_commits: + fail("manifest names an unreviewed upstream commit") + for entry in entries: + validate_entry_shape(entry) + return entries + + +def validate_entry_shape(entry: Entry) -> None: + source_count = len(entry.sources) + expected_count = { + "A64_MACHO_ECDSA_V1": {1}, + "A64_ELF_ECDSA_SPLIT_V1": {1}, + "X86_ELF_ECDSA_GLOBAL_V1": {1}, + "A64_MACHO_CONCAT_TOKEN_V1": {1, 2}, + "A64_ELF_ED25519_GLOBAL_V1": {1, 2}, + "X86_ELF_ED25519_GLOBAL_V1": {1}, + "A64_ELF_X25519_TOKEN_V1": {2}, + "X86_ELF_X25519_CONCAT_TOKEN_V1": {4}, + }[entry.transform] + if source_count not in expected_count: + fail( + f"{entry.local_path} has {source_count} sources for {entry.transform}; " + f"expected {sorted(expected_count)}" + ) + if entry.transform == "X86_ELF_ED25519_GLOBAL_V1": + if entry.commit != ED25519_X86_COMMIT: + fail(f"{entry.local_path} does not use the exact x86 Ed25519 ancestor") + elif entry.transform == "X86_ELF_X25519_CONCAT_TOKEN_V1": + if entry.commit != X25519_X86_COMMIT: + fail(f"{entry.local_path} does not use the exact x86 X25519 ancestor") + elif entry.commit != PINNED_COMMIT: + fail(f"{entry.local_path} does not use the pinned upstream commit") + + +def generated_body(data: bytes, name: str) -> bytes: + match = BODY_RE.search(data) + if match is None: + fail(f"{name} has no generated .globl body") + return data[match.start() :] + + +def external_signature_snapshots() -> set[str]: + roots = ( + ROOT / "src/auth/asm", + ROOT / "src/auth/ed25519/asm", + ROOT / "src/auth/x25519/asm", + ) + snapshots: set[str] = set() + for directory in roots: + for path in directory.rglob("*"): + if not path.is_file() or path.suffix not in {".s", ".S"}: + continue + data = read_file(path) + external = ( + b"Adapted for rscrypto" in data + or b"The butterfly schedule is auto-derived from" in data + ) + if external and not path.name.startswith("rscrypto_rsa_"): + snapshots.add(path.relative_to(ROOT).as_posix()) + return snapshots + + +def verify_local(entries: list[Entry]) -> None: + manifest_paths = {entry.local_path for entry in entries} + snapshots = external_signature_snapshots() + if manifest_paths != snapshots: + missing = sorted(snapshots - manifest_paths) + extra = sorted(manifest_paths - snapshots) + fail(f"manifest coverage drift; missing={missing}, extra={extra}") + + for entry in entries: + data = read_file(ROOT / entry.local_path) + actual = digest(data) + if actual != entry.local_sha256: + fail( + f"{entry.local_path} has SHA-256 {actual}; " + f"expected {entry.local_sha256}" + ) + body = generated_body(data, entry.local_path) + actual_generated = digest(body) + if actual_generated != entry.generated_sha256: + fail( + f"{entry.local_path} generated body has SHA-256 {actual_generated}; " + f"expected {entry.generated_sha256}" + ) + + +class Upstream: + def __init__(self, repository: Path, clang: str, temporary_root: Path) -> None: + self.repository = repository + self.clang = clang + self.temporary_root = temporary_root + self.blobs: dict[tuple[str, str], bytes] = {} + self.include_dirs: dict[tuple[str, str], Path] = {} + + def git(self, *args: str) -> bytes: + try: + result = subprocess.run( + ["git", "-C", str(self.repository), *args], + check=True, + capture_output=True, + ) + except OSError as error: + fail(f"cannot execute git: {error}") + except subprocess.CalledProcessError as error: + detail = error.stderr.decode("utf-8", "replace").strip() + fail(f"git {' '.join(args)} failed: {detail}") + return result.stdout + + def verify_identity(self) -> None: + inside = self.git("rev-parse", "--is-inside-work-tree").decode().strip() + if inside != "true": + fail(f"{self.repository} is not a Git worktree") + remote = self.git("remote", "get-url", "origin").decode().strip() + if remote.rstrip("/") != UPSTREAM_URL.rstrip("/"): + fail(f"upstream origin is {remote}; expected {UPSTREAM_URL}") + for commit in (PINNED_COMMIT, ED25519_X86_COMMIT, X25519_X86_COMMIT): + actual = self.git("rev-parse", f"{commit}^{{commit}}").decode().strip() + if actual != commit: + fail(f"upstream repository does not contain exact commit {commit}") + + def blob(self, commit: str, member: str) -> bytes: + key = (commit, member) + if key not in self.blobs: + self.blobs[key] = self.git("show", f"{commit}:{member}") + return self.blobs[key] + + def include_dir(self, commit: str, architecture: str) -> Path: + key = (commit, architecture) + if key in self.include_dirs: + return self.include_dirs[key] + header = f"_internal_s2n_bignum_{architecture}.h" + directory = self.temporary_root / commit / "include" + directory.mkdir(parents=True, exist_ok=True) + try: + (directory / header).write_bytes(self.blob(commit, f"include/{header}")) + except OSError as error: + fail(f"cannot materialize upstream preprocessor header: {error}") + self.include_dirs[key] = directory + return directory + + def preprocess( + self, + source: bytes, + commit: str, + architecture: str, + target: str, + *, + line_markers: bool, + strip_comments: bool, + ) -> bytes: + if strip_comments: + source = re.sub(rb"//[^\n]*", b"", source) + command = [ + self.clang, + f"--target={target}", + "-E", + ] + if not line_markers: + command.append("-P") + command.extend( + [ + f"-I{self.include_dir(commit, architecture)}", + "-DS2N_BN_HIDE_SYMBOLS=1", + "-x", + "assembler-with-cpp", + "-", + ] + ) + environment = os.environ.copy() + environment["LC_ALL"] = "C" + try: + result = subprocess.run( + command, + input=source, + check=True, + capture_output=True, + env=environment, + ) + except OSError as error: + fail(f"cannot execute {self.clang}: {error}") + except subprocess.CalledProcessError as error: + detail = error.stderr.decode("utf-8", "replace").strip() + fail(f"preprocessing failed for {target}: {detail}") + if b"\r" in result.stdout: + fail(f"{self.clang} emitted non-LF output for {target}") + return result.stdout + + +def public_symbols(data: bytes) -> tuple[bytes, ...]: + symbols = tuple(dict.fromkeys(GLOBAL_RE.findall(data))) + if not symbols: + fail("preprocessed assembly exports no symbols") + return symbols + + +def source_symbols(source: bytes, preprocessed: bytes) -> tuple[bytes, ...]: + public = public_symbols(preprocessed) + macho = public[0].startswith(b"_") + declared = ( + (b"_" + symbol if macho else symbol) + for symbol in SOURCE_SYMBOL_RE.findall(source) + ) + return tuple(dict.fromkeys((*public, *declared))) + + +def prefixed(symbol: bytes) -> bytes: + if symbol.startswith(b"_"): + return b"_rscrypto_" + symbol[1:] + return b"rscrypto_" + symbol + + +def exact_token_rename(data: bytes, symbols: tuple[bytes, ...]) -> bytes: + alternatives = b"|".join(re.escape(symbol) for symbol in sorted(symbols, key=len, reverse=True)) + pattern = re.compile( + rb"(? bytes: + alternatives = b"|".join(re.escape(symbol) for symbol in sorted(symbols, key=len, reverse=True)) + pattern = re.compile(alternatives) + return pattern.sub(lambda match: prefixed(match.group(0)), data) + + +def delete_line_markers(data: bytes) -> bytes: + return b"".join( + line + for line in data.splitlines(keepends=True) + if not line.lstrip().startswith(b"#") + ) + + +def delete_gnu_stack(data: bytes) -> bytes: + return re.sub( + rb"(?m)^[ \t]*\.section[ \t]+\.note\.GNU-stack[^\n]*(?:\n|$)", + b"", + data, + ) + + +def one_source(sources: tuple[bytes, ...], transform: str) -> bytes: + if len(sources) != 1: + fail(f"{transform} requires one source") + return sources[0] + + +def a64_macho_ecdsa( + upstream: Upstream, entry: Entry, sources: tuple[bytes, ...] +) -> bytes: + output = upstream.preprocess( + one_source(sources, entry.transform), + entry.commit, + "arm", + "arm64-apple-darwin", + line_markers=True, + strip_comments=True, + ) + output = delete_line_markers(output) + output = exact_token_rename(output, public_symbols(output)) + return generated_body(output, entry.transform) + + +def a64_elf_ecdsa_split( + upstream: Upstream, entry: Entry, sources: tuple[bytes, ...] +) -> bytes: + output = upstream.preprocess( + one_source(sources, entry.transform), + entry.commit, + "arm", + "aarch64-linux-gnu", + line_markers=True, + strip_comments=False, + ) + output = output.replace(b" ; ", b"; ").replace(b";", b"\n") + lines: list[bytes] = [] + for line in delete_line_markers(output).splitlines(): + stripped = line.lstrip() + if stripped.startswith((b".type ", b".size ")): + lines.append(b"") + elif stripped.startswith(b".section .note.GNU-stack"): + continue + else: + lines.append(line.rstrip()) + while lines and not lines[-1]: + lines.pop() + output = b"\n".join(lines) + b"\n" + output = exact_token_rename(output, public_symbols(output)) + return generated_body(output, entry.transform) + + +def x86_elf_ecdsa_global( + upstream: Upstream, entry: Entry, sources: tuple[bytes, ...] +) -> bytes: + output = upstream.preprocess( + one_source(sources, entry.transform), + entry.commit, + "x86_att", + "x86_64-linux-gnu", + line_markers=False, + strip_comments=False, + ) + output = global_stem_rename(output, public_symbols(output)) + output = delete_gnu_stack(output).rstrip(b"\n") + b"\n" + return generated_body(output, entry.transform) + + +def a64_macho_concat_token( + upstream: Upstream, entry: Entry, sources: tuple[bytes, ...] +) -> bytes: + parts: list[bytes] = [] + for source in sources: + output = upstream.preprocess( + source, + entry.commit, + "arm", + "arm64-apple-darwin", + line_markers=False, + strip_comments=True, + ) + output = exact_token_rename(output, source_symbols(source, output)) + parts.append(generated_body(output, entry.transform).rstrip(b"\n")) + return b"\n\n\n".join(parts) + b"\n" + + +def a64_elf_ed25519_global( + upstream: Upstream, entry: Entry, sources: tuple[bytes, ...] +) -> bytes: + parts: list[bytes] = [] + for source in sources: + output = upstream.preprocess( + source, + entry.commit, + "arm", + "aarch64-linux-gnu", + line_markers=False, + strip_comments=False, + ) + output = global_stem_rename(output, public_symbols(output)) + parts.append(generated_body(output, entry.transform).rstrip(b"\n")) + return b"\n\n\n\n".join(parts) + b"\n" + + +def x86_elf_ed25519_global( + upstream: Upstream, entry: Entry, sources: tuple[bytes, ...] +) -> bytes: + output = upstream.preprocess( + one_source(sources, entry.transform), + entry.commit, + "x86_att", + "x86_64-linux-gnu", + line_markers=False, + strip_comments=False, + ) + output = global_stem_rename(output, public_symbols(output)) + return generated_body(output, entry.transform) + + +def a64_elf_x25519_token( + upstream: Upstream, entry: Entry, sources: tuple[bytes, ...] +) -> bytes: + parts: list[bytes] = [] + for source in sources: + output = upstream.preprocess( + source, + entry.commit, + "arm", + "aarch64-linux-gnu", + line_markers=False, + strip_comments=False, + ) + symbols = source_symbols(source, output) + output = exact_token_rename(output, symbols) + output = delete_gnu_stack(output) + for symbol in symbols: + renamed = re.escape(prefixed(symbol)) + if symbol.endswith(b"_constant"): + output = re.sub( + rb"(?m)^[ \t]*\.(?:type|size)[ \t]+" + + renamed + + rb"[^\n]*(?:\n|$)", + b"", + output, + ) + parts.append(generated_body(output, entry.transform).rstrip(b"\n")) + return b"\n\n\n".join(parts) + b"\n" + + +def x86_elf_x25519_concat_token( + upstream: Upstream, entry: Entry, sources: tuple[bytes, ...] +) -> bytes: + parts: list[bytes] = [] + for source in sources: + output = upstream.preprocess( + source, + entry.commit, + "x86_att", + "x86_64-linux-gnu", + line_markers=False, + strip_comments=False, + ) + output = exact_token_rename(output, source_symbols(source, output)) + output = delete_gnu_stack(output) + parts.append(generated_body(output, entry.transform).rstrip(b"\n")) + return ( + b"\n\n".join(parts) + + b'\n\n.section .note.GNU-stack, "", %progbits\n' + ) + + +Transform = Callable[[Upstream, Entry, tuple[bytes, ...]], bytes] +TRANSFORMS: dict[str, Transform] = { + "A64_MACHO_ECDSA_V1": a64_macho_ecdsa, + "A64_ELF_ECDSA_SPLIT_V1": a64_elf_ecdsa_split, + "X86_ELF_ECDSA_GLOBAL_V1": x86_elf_ecdsa_global, + "A64_MACHO_CONCAT_TOKEN_V1": a64_macho_concat_token, + "A64_ELF_ED25519_GLOBAL_V1": a64_elf_ed25519_global, + "X86_ELF_ED25519_GLOBAL_V1": x86_elf_ed25519_global, + "A64_ELF_X25519_TOKEN_V1": a64_elf_x25519_token, + "X86_ELF_X25519_CONCAT_TOKEN_V1": x86_elf_x25519_concat_token, +} + + +def verify_upstream(entries: list[Entry], repository: Path, clang: str) -> None: + with tempfile.TemporaryDirectory(prefix="rscrypto-signature-asm-") as temporary: + upstream = Upstream(repository, clang, Path(temporary)) + upstream.verify_identity() + for entry in entries: + source_data: list[bytes] = [] + for source in entry.sources: + data = upstream.blob(entry.commit, source.member) + actual = digest(data) + if actual != source.sha256: + fail( + f"{entry.commit}:{source.member} has SHA-256 {actual}; " + f"expected {source.sha256}" + ) + source_data.append(data) + generated = TRANSFORMS[entry.transform]( + upstream, entry, tuple(source_data) + ) + actual = digest(generated) + if actual != entry.generated_sha256: + fail( + f"{entry.local_path} regenerated body has SHA-256 {actual}; " + f"expected {entry.generated_sha256}" + ) + local = generated_body(read_file(ROOT / entry.local_path), entry.local_path) + if generated != local: + fail(f"{entry.local_path} differs from regenerated upstream body") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Verify non-RSA signature assembly provenance." + ) + parser.add_argument( + "--upstream-repo", + type=Path, + help="s2n-bignum Git worktree used for source and regeneration checks", + ) + parser.add_argument( + "--clang", + default=os.environ.get("CLANG", "clang"), + help="C preprocessor driver for --upstream-repo mode (default: clang)", + ) + args = parser.parse_args() + + entries = parse_manifest() + verify_local(entries) + if args.upstream_repo is not None: + verify_upstream(entries, args.upstream_repo, args.clang) + print( + "signature-asm-provenance: 36 snapshots match immutable upstream " + "sources and transforms" + ) + else: + print( + "signature-asm-provenance: 36 snapshots match the provenance " + "manifest" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/check/zeroize-evidence.sh b/scripts/check/zeroize-evidence.sh index ba6833db..0dd6a8d1 100755 --- a/scripts/check/zeroize-evidence.sh +++ b/scripts/check/zeroize-evidence.sh @@ -11,7 +11,7 @@ RUSTC_WRAPPER="" CARGO_TARGET_DIR="$TARGET_DIR" cargo rustc \ --release \ --lib \ --no-default-features \ - --features alloc,aes-gcm,blake3,hmac,hmac-sha3,parallel,diag \ + --features alloc,aegis256,aes-gcm,ascon-aead,blake3,chacha20poly1305,ecdsa-p256,ecdsa-p384,hkdf,hmac,hmac-sha3,ml-kem,parallel,rsa,diag \ -- \ -Ccodegen-units=1 \ --emit=mir,llvm-ir,asm @@ -53,7 +53,19 @@ for symbol in \ diag_zeroize_blake3_thread_scratch \ diag_zeroize_blake3_parallel_scratch \ diag_zeroize_hmac_sha256_finalize \ - diag_zeroize_hmac_sha3_finalize; do + diag_zeroize_hmac_sha3_finalize \ + diag_hkdf_sha256_derive_portable \ + diag_hkdf_sha384_derive_portable \ + diag_hkdf_sha512_derive_portable \ + diag_poly1305_block_portable_digest \ + diag_ascon_aead128_tag_portable \ + diag_aegis256_update_portable \ + diag_aes128gcm_ghash \ + diag_aes256gcm_ghash \ + diag_zeroize_mlkem_sha3_512 \ + diag_zeroize_mlkem_shake256_scalar \ + diag_zeroize_mlkem_shake256_pair \ + diag_zeroize_mlkem_shake256_quad; do if ! grep -q "@$symbol" "$LLVM_IR"; then echo "zeroize LLVM evidence missing symbol: $symbol" >&2 exit 1 @@ -82,7 +94,19 @@ for symbol in \ diag_zeroize_blake3_thread_scratch \ diag_zeroize_blake3_parallel_scratch \ diag_zeroize_hmac_sha256_finalize \ - diag_zeroize_hmac_sha3_finalize; do + diag_zeroize_hmac_sha3_finalize \ + diag_hkdf_sha256_derive_portable \ + diag_hkdf_sha384_derive_portable \ + diag_hkdf_sha512_derive_portable \ + diag_poly1305_block_portable_digest \ + diag_ascon_aead128_tag_portable \ + diag_aegis256_update_portable \ + diag_aes128gcm_ghash \ + diag_aes256gcm_ghash \ + diag_zeroize_mlkem_sha3_512 \ + diag_zeroize_mlkem_shake256_scalar \ + diag_zeroize_mlkem_shake256_pair \ + diag_zeroize_mlkem_shake256_quad; do FUNCTION_IR="$(sed -n "/define .*@$symbol(/,/^}/p" "$LLVM_IR")" VOLATILE_STORES="$(grep -c 'store volatile .* 0' <<<"$FUNCTION_IR" || true)" if [[ "$VOLATILE_STORES" -lt 1 ]]; then @@ -91,8 +115,90 @@ for symbol in \ fi done +POLY1305_IR="$(sed -n '/define .*@diag_poly1305_block_portable_digest(/,/^}/p' "$LLVM_IR")" +if [[ "$(grep -c 'store volatile i32 0' <<<"$POLY1305_IR" || true)" -lt 14 ]] || \ + ! grep -q 'fence syncscope("singlethread") seq_cst' <<<"$POLY1305_IR"; then + echo "zeroize release evidence does not clear the complete portable Poly1305 state" >&2 + exit 1 +fi + +ASCON_IR="$(sed -n '/define .*@diag_ascon_aead128_tag_portable(/,/^}/p' "$LLVM_IR")" +ASCON_STATE_CLEANUP="$(sed -n '1,/fence syncscope("singlethread") seq_cst/p' <<<"$ASCON_IR")" +if [[ "$(grep -c 'store volatile i64 0' <<<"$ASCON_STATE_CLEANUP" || true)" -lt 5 ]] || \ + ! grep -q 'fence syncscope("singlethread") seq_cst' <<<"$ASCON_IR"; then + echo "zeroize release evidence does not clear the complete portable Ascon-AEAD state" >&2 + exit 1 +fi + +AEGIS_IR="$(sed -n '/define .*@diag_aegis256_update_portable(/,/^}/p' "$LLVM_IR")" +if [[ "$(grep -c 'store volatile .* 0' <<<"$AEGIS_IR" || true)" -lt 3 ]] || \ + ! grep -q 'fence syncscope("singlethread") seq_cst' <<<"$AEGIS_IR"; then + echo "zeroize release evidence does not retain the portable AEGIS-256 state wipe" >&2 + exit 1 +fi + +for symbol in diag_aes128gcm_ghash diag_aes256gcm_ghash; do + FUNCTION_IR="$(sed -n "/define .*@$symbol(/,/^}/p" "$LLVM_IR")" + if ! grep -q 'store volatile i128 0' <<<"$FUNCTION_IR" || \ + ! grep -q 'fence syncscope("singlethread") seq_cst' <<<"$FUNCTION_IR"; then + echo "zeroize release evidence does not retain the GCM authentication-state wipe in $symbol" >&2 + exit 1 + fi +done + +MLKEM_SHA3_IR="$(sed -n '/define .*@diag_zeroize_mlkem_sha3_512(/,/^}/p' "$LLVM_IR")" +MLKEM_SHA3_FINALIZE_SYMBOL="$( + sed -n 's/.*call .*@\([^ (]*KeccakCoreImpl[^ (]*finalize_into_fixed[^ (]*\).*/\1/p' \ + <<<"$MLKEM_SHA3_IR" | head -n 1 +)" +MLKEM_SHA3_FINALIZE_IR="$(sed -n "/define .*@$MLKEM_SHA3_FINALIZE_SYMBOL(/,/^}/p" "$LLVM_IR")" +if [[ "$(grep -c 'store volatile i64 0' <<<"$MLKEM_SHA3_IR" || true)" -lt 25 ]] || \ + [[ "$(grep -c 'store volatile i64 0' <<<"$MLKEM_SHA3_FINALIZE_IR" || true)" -lt 25 ]] || \ + ! grep -q 'fence syncscope("singlethread") seq_cst' <<<"$MLKEM_SHA3_IR" || \ + ! grep -q 'fence syncscope("singlethread") seq_cst' <<<"$MLKEM_SHA3_FINALIZE_IR"; then + echo "zeroize release evidence does not clear ML-KEM SHA3-512 owner and finalization states" >&2 + exit 1 +fi + +MLKEM_SHAKE_SCALAR_IR="$(sed -n '/define .*@diag_zeroize_mlkem_shake256_scalar(/,/^}/p' "$LLVM_IR")" +MLKEM_SHAKE_SCALAR_SEED_SYMBOL="$( + grep 'call .*MlKemShake256XofReader.*seeded_32_1' <<<"$MLKEM_SHAKE_SCALAR_IR" | + grep -v 'quad' | + sed -n 's/.*@\([^ (]*\).*/\1/p' | + head -n 1 +)" +MLKEM_SHAKE_SCALAR_SEED_IR="$(sed -n "/define .*@$MLKEM_SHAKE_SCALAR_SEED_SYMBOL(/,/^}/p" "$LLVM_IR")" +if [[ "$(grep -c 'store volatile i64 0' <<<"$MLKEM_SHAKE_SCALAR_IR" || true)" -lt 25 ]] || \ + [[ "$(grep -c 'store volatile i64 0' <<<"$MLKEM_SHAKE_SCALAR_SEED_IR" || true)" -lt 25 ]] || \ + ! grep -q 'fence syncscope("singlethread") seq_cst' <<<"$MLKEM_SHAKE_SCALAR_IR" || \ + ! grep -q 'fence syncscope("singlethread") seq_cst' <<<"$MLKEM_SHAKE_SCALAR_SEED_IR"; then + echo "zeroize release evidence does not clear ML-KEM scalar SHAKE owner and seeded state" >&2 + exit 1 +fi + +MLKEM_SHAKE_PAIR_IR="$(sed -n '/define .*@diag_zeroize_mlkem_shake256_pair(/,/^}/p' "$LLVM_IR")" +if [[ "$(grep -c 'store volatile i64 0' <<<"$MLKEM_SHAKE_PAIR_IR" || true)" -lt 100 ]] || \ + ! grep -q 'fence syncscope("singlethread") seq_cst' <<<"$MLKEM_SHAKE_PAIR_IR"; then + echo "zeroize release evidence does not clear both ML-KEM pair owners and seeded states" >&2 + exit 1 +fi + +MLKEM_SHAKE_QUAD_IR="$(sed -n '/define .*@diag_zeroize_mlkem_shake256_quad(/,/^}/p' "$LLVM_IR")" +MLKEM_SHAKE_QUAD_SEED_SYMBOL="$( + sed -n 's/.*call .*@\([^ (]*MlKemShake256XofReader[^ (]*seeded_32_1_quad[^ (]*\).*/\1/p' \ + <<<"$MLKEM_SHAKE_QUAD_IR" | head -n 1 +)" +MLKEM_SHAKE_QUAD_SEED_IR="$(sed -n "/define .*@$MLKEM_SHAKE_QUAD_SEED_SYMBOL(/,/^}/p" "$LLVM_IR")" +if [[ "$(grep -c 'store volatile i64 0' <<<"$MLKEM_SHAKE_QUAD_IR" || true)" -lt 100 ]] || \ + [[ "$(grep -c 'store volatile i64 0' <<<"$MLKEM_SHAKE_QUAD_SEED_IR" || true)" -lt 100 ]] || \ + ! grep -q 'fence syncscope("singlethread") seq_cst' <<<"$MLKEM_SHAKE_QUAD_IR" || \ + ! grep -q 'fence syncscope("singlethread") seq_cst' <<<"$MLKEM_SHAKE_QUAD_SEED_IR"; then + echo "zeroize release evidence does not clear all ML-KEM quad owners and seeded states" >&2 + exit 1 +fi + BLAKE3_DROP_WRAPPER="$(sed -n '/define .*@diag_zeroize_blake3_drop(/,/^}/p' "$LLVM_IR")" -BLAKE3_DROP_SYMBOL="$(sed -n 's/.*call .*@\([^ (]*drop_in_place[^ (]*Blake3[^ (]*\).*/\1/p' \ +BLAKE3_DROP_SYMBOL="$(sed -En 's/.*call .*@([^ (]*drop_(in_place|glue)[^ (]*Blake3[^ (]*).*/\1/p' \ <<<"$BLAKE3_DROP_WRAPPER" | head -n 1)" if [[ -z "$BLAKE3_DROP_SYMBOL" ]]; then echo "zeroize LLVM evidence does not route BLAKE3 cleanup through its production Drop" >&2 @@ -107,7 +213,7 @@ if [[ "$BLAKE3_DROP_STORES" -lt 8 ]] || ! grep -q "$BLAKE3_DROP_SYMBOL" "$ASSEMB fi BLAKE3_REUSE_IR="$(sed -n '/define .*@diag_zeroize_blake3_reuse(/,/^}/p' "$LLVM_IR")" -BLAKE3_REUSE_DROPS="$(grep -c '^[[:space:]]*call .*drop_in_place.*Blake3' <<<"$BLAKE3_REUSE_IR" || true)" +BLAKE3_REUSE_DROPS="$(grep -Ec '^[[:space:]]*call .*drop_(in_place|glue).*Blake3' <<<"$BLAKE3_REUSE_IR" || true)" if [[ "$BLAKE3_REUSE_DROPS" -lt 2 ]]; then echo "zeroize release evidence does not wipe both replaced and final BLAKE3 state" >&2 exit 1 @@ -161,6 +267,56 @@ if [[ "$(grep -c 'store volatile .* 0' <<<"$HMAC_SHA256_SECRET_FINALIZE_IR" || t exit 1 fi +for symbol in \ + diag_hkdf_sha256_derive_portable \ + diag_hkdf_sha384_derive_portable \ + diag_hkdf_sha512_derive_portable; do + FUNCTION_IR="$(sed -n "/define .*@$symbol(/,/^}/p" "$LLVM_IR")" + if [[ "$(grep -c 'store volatile .* 0' <<<"$FUNCTION_IR" || true)" -lt 8 ]] || \ + ! grep -q 'fence syncscope("singlethread") seq_cst' <<<"$FUNCTION_IR"; then + echo "zeroize release evidence does not retain HKDF prefix and expansion-scratch cleanup in $symbol" >&2 + exit 1 + fi +done + +ECDSA_P256_IR="$(sed -n '/define .*@diag_zeroize_ecdsa_p256_platform_scratch(/,/^}/p' "$LLVM_IR")" +if [[ -z "$ECDSA_P256_IR" ]] || \ + ! grep -q 'diag_zeroize_ecdsa_p256_platform_scratch' "$MIR" || \ + ! grep -q 'diag_zeroize_ecdsa_p256_platform_scratch' "$ASSEMBLY" || \ + [[ "$(grep -c 'store volatile i64 0' <<<"$ECDSA_P256_IR" || true)" -lt 26 ]] || \ + ! grep -q 'fence syncscope("singlethread") seq_cst' <<<"$ECDSA_P256_IR"; then + echo "zeroize release evidence does not clear P-256 accelerated reduction and inversion scratch" >&2 + exit 1 +fi + +if [[ "$(rustc -vV | sed -n 's/^host: //p')" == aarch64-* ]]; then + ECDSA_P384_IR="$(sed -n '/define .*@diag_zeroize_ecdsa_p384_platform_scratch(/,/^}/p' "$LLVM_IR")" + ECDSA_P384_INVERSE_CLEANUP="$( + awk '/fence syncscope\("singlethread"\) seq_cst/{fences++; next} fences == 0' \ + <<<"$ECDSA_P384_IR" + )" + ECDSA_P384_REDUCED_CLEANUP="$( + awk '/fence syncscope\("singlethread"\) seq_cst/{fences++; next} fences == 1' \ + <<<"$ECDSA_P384_IR" + )" + ECDSA_P384_WIDE_CLEANUP="$( + awk '/fence syncscope\("singlethread"\) seq_cst/{fences++; next} fences >= 2' \ + <<<"$ECDSA_P384_IR" + )" + if [[ -z "$ECDSA_P384_IR" ]] || \ + ! grep -q 'diag_zeroize_ecdsa_p384_platform_scratch' "$MIR" || \ + ! grep -q 'diag_zeroize_ecdsa_p384_platform_scratch' "$ASSEMBLY" || \ + [[ "$(grep -c 'store volatile i64 0' <<<"$ECDSA_P384_INVERSE_CLEANUP" || true)" -lt 6 ]] || \ + [[ "$(grep -c 'store volatile i64 0' <<<"$ECDSA_P384_REDUCED_CLEANUP" || true)" -lt 6 ]] || \ + ! grep -q 'sub nuw nsw i64 96' <<<"$ECDSA_P384_WIDE_CLEANUP" || \ + ! grep -q 'store volatile i64 0' <<<"$ECDSA_P384_WIDE_CLEANUP" || \ + ! grep -q 'store volatile i8 0' <<<"$ECDSA_P384_WIDE_CLEANUP" || \ + [[ "$(grep -c 'fence syncscope("singlethread") seq_cst' <<<"$ECDSA_P384_IR" || true)" -lt 3 ]]; then + echo "zeroize release evidence does not clear P-384 accelerated input, reduction, and inversion scratch" >&2 + exit 1 + fi +fi + HEX_ERROR_IR="$(sed -n '/define .*@diag_zeroize_hex_error(/,/^}/p' "$LLVM_IR")" HEX_SUCCESS_IR="$(sed -n '/define .*@diag_zeroize_hex_success(/,/^}/p' "$LLVM_IR")" HEX_FROM_STR="$(sed -n 's/.*call .*@\([^ (]*FromStr8from_str\).*/\1/p' <<<"$HEX_ERROR_IR" | head -n 1)" @@ -176,22 +332,203 @@ if [[ "$HEX_VOLATILE_STORES" -lt 2 ]]; then exit 1 fi +RSA_VALIDATION_WRAPPER_IR="$( + sed -n '/define .*diag_rsa_validate_pkcs8_private_key_der_stage(/,/^}/p' "$LLVM_IR" +)" +llvm_calls() { + local symbol_pattern="$1" + grep -E \ + "^[[:space:]]*(%[^=]+=[[:space:]]*)?((musttail|tail|notail)[[:space:]]+)?call[[:space:]].*$symbol_pattern" || + true +} + +RSA_VALIDATION_SYMBOL="$( + llvm_calls 'validate_private_key_components_through_stage' \ + <<<"$RSA_VALIDATION_WRAPPER_IR" | + sed -n 's/.*@\([^ (]*validate_private_key_components_through_stage[^ (]*\).*/\1/p' | + head -n 1 +)" +if [[ -z "$RSA_VALIDATION_SYMBOL" ]] || \ + ! grep -q 'diag_rsa_validate_pkcs8_private_key_der_stage' "$MIR" || \ + ! grep -q 'diag_rsa_validate_pkcs8_private_key_der_stage' "$ASSEMBLY"; then + echo "zeroize RSA private-key validation evidence is missing" >&2 + exit 1 +fi + +RSA_VALIDATION_IR="$(sed -n "/define .*@$RSA_VALIDATION_SYMBOL(/,/^}/p" "$LLVM_IR")" +RSA_SECRET_OWNER_CONSTRUCTION_CALLS="$( + llvm_calls 'SecretBigEndianBuffer.*zeroed' <<<"$RSA_VALIDATION_IR" +)" +RSA_SECRET_OWNER_DROP_CALLS="$( + llvm_calls 'drop_(in_place|glue).*SecretBigEndianBuffer' <<<"$RSA_VALIDATION_IR" +)" +RSA_SECRET_OWNER_CONSTRUCTIONS="$(grep -c . <<<"$RSA_SECRET_OWNER_CONSTRUCTION_CALLS" || true)" +RSA_SECRET_OWNER_DROPS="$(grep -c . <<<"$RSA_SECRET_OWNER_DROP_CALLS" || true)" +RSA_SECRET_CONSTRUCTION_OPERANDS="$( + sed -n 's/.*(ptr [^%]*\(%[^,)]*\).*/\1/p' <<<"$RSA_SECRET_OWNER_CONSTRUCTION_CALLS" | + sort -u | + grep -c . || true +)" +RSA_SECRET_DROP_OPERANDS="$( + sed -n 's/.*(ptr [^%]*\(%[^,)]*\).*/\1/p' <<<"$RSA_SECRET_OWNER_DROP_CALLS" | + sort -u | + grep -c . || true +)" +RSA_SECRET_DROP_SYMBOL="$( + sed -En 's/.*@([^ (]*drop_(in_place|glue)[^ (]*SecretBigEndianBuffer[^ (]*).*/\1/p' \ + <<<"$RSA_SECRET_OWNER_DROP_CALLS" | head -n 1 +)" +if [[ "$RSA_SECRET_OWNER_CONSTRUCTIONS" -ne 9 || "$RSA_SECRET_OWNER_DROPS" -ne 9 || \ + "$RSA_SECRET_CONSTRUCTION_OPERANDS" -ne 9 || "$RSA_SECRET_DROP_OPERANDS" -ne 9 || \ + -z "$RSA_SECRET_DROP_SYMBOL" ]] || \ + llvm_calls '__rust_dealloc' <<<"$RSA_VALIDATION_IR" | grep -q .; then + echo "zeroize RSA private-key validation does not retain all RAII cleanup paths" >&2 + exit 1 +fi + +RSA_VALIDATION_WITHOUT_ONE_CONSTRUCTION="$( + awk ' + !removed && + /^[[:space:]]*(%[^=]+=[[:space:]]*)?((musttail|tail|notail)[[:space:]]+)?call[[:space:]]/ && + /SecretBigEndianBuffer.*zeroed/ { + removed = 1 + next + } + { print } + ' <<<"$RSA_VALIDATION_IR" +)" +RSA_VALIDATION_WITHOUT_ONE_DROP="$( + awk ' + !removed && + /^[[:space:]]*(%[^=]+=[[:space:]]*)?((musttail|tail|notail)[[:space:]]+)?call[[:space:]]/ && + /drop_(in_place|glue).*SecretBigEndianBuffer/ { + removed = 1 + next + } + { print } + ' <<<"$RSA_VALIDATION_IR" +)" +if [[ "$(llvm_calls 'SecretBigEndianBuffer.*zeroed' \ + <<<"$RSA_VALIDATION_WITHOUT_ONE_CONSTRUCTION" | grep -c .)" -ne 8 ]] || \ + [[ "$(llvm_calls 'drop_(in_place|glue).*SecretBigEndianBuffer' \ + <<<"$RSA_VALIDATION_WITHOUT_ONE_DROP" | grep -c .)" -ne 8 ]]; then + echo "zeroize RSA private-key validation call parser does not reject a missing owner" >&2 + exit 1 +fi + +RSA_SECRET_DROP_IR="$(sed -n "/define .*@$RSA_SECRET_DROP_SYMBOL(/,/^}/p" "$LLVM_IR")" +RSA_SECRET_LAST_ZERO_LINE="$( + grep -n '^[[:space:]]*store volatile .* 0' <<<"$RSA_SECRET_DROP_IR" | + tail -n 1 | + cut -d: -f1 +)" +RSA_SECRET_FENCE_LINE="$( + grep -n '^[[:space:]]*fence[[:space:]]' <<<"$RSA_SECRET_DROP_IR" | + head -n 1 | + cut -d: -f1 +)" +RSA_SECRET_DEALLOC_LINE="$( + grep -nE \ + '^[[:space:]]*(%[^=]+=[[:space:]]*)?((musttail|tail|notail)[[:space:]]+)?call[[:space:]].*__rust_dealloc' \ + <<<"$RSA_SECRET_DROP_IR" | + head -n 1 | + cut -d: -f1 +)" +if [[ "$(grep -c '^[[:space:]]*store volatile .* 0' <<<"$RSA_SECRET_DROP_IR" || true)" -lt 3 || \ + -z "$RSA_SECRET_LAST_ZERO_LINE" || -z "$RSA_SECRET_FENCE_LINE" || -z "$RSA_SECRET_DEALLOC_LINE" || \ + "$RSA_SECRET_LAST_ZERO_LINE" -ge "$RSA_SECRET_FENCE_LINE" || \ + "$RSA_SECRET_FENCE_LINE" -ge "$RSA_SECRET_DEALLOC_LINE" ]]; then + echo "zeroize RSA private-key validation owner does not wipe before deallocation" >&2 + exit 1 +fi + function_assembly() { local symbol="$1" awk -v plain="$symbol:" -v apple="_$symbol:" ' $0 == plain || $0 == apple { found = 1 } - found && emitted && $0 ~ /^[[:space:]]*\.globl[[:space:]]/ { exit } + found && emitted && $0 ~ /^[^[:space:].Ll][^:]*:$/ { exit } found { print } found { emitted = 1 } - ' "$ASSEMBLY" + ' } -FIXED_ASSEMBLY="$(function_assembly diag_zeroize_fixed_stack)" +AARCH64_ZERO_MEMORY_PATTERN='^[[:space:]]*st(p|u?r)(b|h)?[[:space:]]+(wzr|xzr)(,[[:space:]]*(wzr|xzr))?,[[:space:]]*\[[^]]+\]' +X86_ZERO_MEMORY_PATTERN='^[[:space:]]*mov[bql]?[[:space:]]+\$0,[[:space:]]*[[:alnum:]_+.-]*\([^)]*%[^)]*\)' + +ordered_assembly_cleanup() { + local body="$1" + local zero_pattern="$2" + local zero_line + local barrier_line + local dealloc_line + + zero_line="$(grep -nE "$zero_pattern" <<<"$body" | tail -n 1 | cut -d: -f1)" + barrier_line="$(grep -n 'MEMBARRIER' <<<"$body" | head -n 1 | cut -d: -f1)" + dealloc_line="$( + grep -nE '^[[:space:]]*(b|bl|call|callq|jmp|jmpq)[[:space:]].*__rust_dealloc' <<<"$body" | + head -n 1 | + cut -d: -f1 + )" + + [[ -n "$zero_line" && -n "$barrier_line" && -n "$dealloc_line" && + "$zero_line" -lt "$barrier_line" && "$barrier_line" -lt "$dealloc_line" ]] +} + +FIXED_ASSEMBLY="$(function_assembly diag_zeroize_fixed_stack <"$ASSEMBLY")" +RSA_VALIDATION_ASSEMBLY="$(function_assembly "$RSA_VALIDATION_SYMBOL" <"$ASSEMBLY")" +RSA_SECRET_DROP_ASSEMBLY="$(function_assembly "$RSA_SECRET_DROP_SYMBOL" <"$ASSEMBLY")" +if [[ "$RSA_VALIDATION_ASSEMBLY" != *"$RSA_SECRET_DROP_SYMBOL"* || -z "$RSA_SECRET_DROP_ASSEMBLY" ]]; then + echo "zeroize RSA private-key validation assembly does not retain owner cleanup" >&2 + exit 1 +fi + +RSA_ASSEMBLY_NEGATIVE_FIXTURE="$( + printf '%s\n' \ + "$RSA_SECRET_DROP_SYMBOL:" \ + $'\tret' \ + '_later_zeroizing_function:' \ + $'\tstrb\twzr, [x0]' \ + $'\t;MEMBARRIER' \ + $'\tb\t__rust_dealloc' +)" +RSA_ASSEMBLY_NEGATIVE_BODY="$( + function_assembly "$RSA_SECRET_DROP_SYMBOL" <<<"$RSA_ASSEMBLY_NEGATIVE_FIXTURE" +)" +if [[ "$RSA_ASSEMBLY_NEGATIVE_BODY" == *'_later_zeroizing_function'* ]] || \ + ordered_assembly_cleanup "$RSA_ASSEMBLY_NEGATIVE_BODY" "$AARCH64_ZERO_MEMORY_PATTERN"; then + echo "zeroize assembly function parser does not reject a later function's cleanup" >&2 + exit 1 +fi + +for fixture in \ + $'; strb wzr, [x0]\n;MEMBARRIER\nb __rust_dealloc' \ + $'# movq $0, (%rax)\n#MEMBARRIER\njmp __rust_dealloc' \ + $'movl $0, %eax\n#MEMBARRIER\njmp __rust_dealloc'; do + if ordered_assembly_cleanup "$fixture" "$AARCH64_ZERO_MEMORY_PATTERN" || \ + ordered_assembly_cleanup "$fixture" "$X86_ZERO_MEMORY_PATTERN"; then + echo "zeroize assembly parser accepts a comment or register-only zero" >&2 + exit 1 + fi +done +if ! ordered_assembly_cleanup \ + $'strb wzr, [x0]\n;MEMBARRIER\nb __rust_dealloc' "$AARCH64_ZERO_MEMORY_PATTERN" || \ + ! grep -Eq "$AARCH64_ZERO_MEMORY_PATTERN" <<< $'stur xzr, [x0, #-8]' || \ + ! ordered_assembly_cleanup \ + $'movq $0, 8(%rax)\n#MEMBARRIER\njmp __rust_dealloc' "$X86_ZERO_MEMORY_PATTERN"; then + echo "zeroize assembly parser rejects a valid ordered memory wipe" >&2 + exit 1 +fi + HOST_ARCH="$(rustc -vV | sed -n 's/^host: \([^-]*\).*/\1/p')" case "$HOST_ARCH" in aarch64) + if ! ordered_assembly_cleanup \ + "$RSA_SECRET_DROP_ASSEMBLY" "$AARCH64_ZERO_MEMORY_PATTERN"; then + echo "zeroize RSA private-key validation assembly does not wipe before deallocation" >&2 + exit 1 + fi if ! grep -Eq 'st(p|r)[[:space:]].*\[sp' <<<"$FIXED_ASSEMBLY" || \ - ! grep -Eq 'str(b|h)?[[:space:]].*(wzr|xzr)' <<<"$FIXED_ASSEMBLY"; then + ! grep -Eq "$AARCH64_ZERO_MEMORY_PATTERN" <<<"$FIXED_ASSEMBLY"; then echo "zeroize assembly evidence does not show the fixed-size stack spill and wipe" >&2 exit 1 fi @@ -205,16 +542,36 @@ case "$HOST_ARCH" in diag_zeroize_blake3_thread_scratch \ diag_zeroize_blake3_parallel_scratch \ diag_zeroize_hmac_sha256_finalize \ - diag_zeroize_hmac_sha3_finalize; do - FUNCTION_ASSEMBLY="$(function_assembly "$symbol")" - if ! grep -Eq 'st(p|r)(b|h)?[[:space:]].*(wzr|xzr)' <<<"$FUNCTION_ASSEMBLY"; then + diag_zeroize_hmac_sha3_finalize \ + diag_hkdf_sha256_derive_portable \ + diag_hkdf_sha384_derive_portable \ + diag_hkdf_sha512_derive_portable \ + diag_poly1305_block_portable_digest \ + diag_ascon_aead128_tag_portable \ + diag_aegis256_update_portable \ + diag_aes128gcm_ghash \ + diag_aes256gcm_ghash \ + diag_zeroize_mlkem_sha3_512 \ + diag_zeroize_mlkem_shake256_scalar \ + diag_zeroize_mlkem_shake256_pair \ + diag_zeroize_mlkem_shake256_quad \ + diag_zeroize_ecdsa_p256_platform_scratch \ + diag_zeroize_ecdsa_p384_platform_scratch; do + FUNCTION_ASSEMBLY="$(function_assembly "$symbol" <"$ASSEMBLY")" + if ! grep -Eq "$AARCH64_ZERO_MEMORY_PATTERN" <<<"$FUNCTION_ASSEMBLY"; then echo "zeroize assembly evidence has no zero store in $symbol" >&2 exit 1 fi done ;; x86_64) - if ! grep -Eq '%rsp' <<<"$FIXED_ASSEMBLY" || ! grep -Eq "mov[bql]?[[:space:]]+\\\$0" <<<"$FIXED_ASSEMBLY"; then + if ! ordered_assembly_cleanup \ + "$RSA_SECRET_DROP_ASSEMBLY" "$X86_ZERO_MEMORY_PATTERN"; then + echo "zeroize RSA private-key validation assembly does not wipe before deallocation" >&2 + exit 1 + fi + if ! grep -Eq '%rsp' <<<"$FIXED_ASSEMBLY" || \ + ! grep -Eq "$X86_ZERO_MEMORY_PATTERN" <<<"$FIXED_ASSEMBLY"; then echo "zeroize assembly evidence does not show the fixed-size stack spill and wipe" >&2 exit 1 fi @@ -228,9 +585,22 @@ case "$HOST_ARCH" in diag_zeroize_blake3_thread_scratch \ diag_zeroize_blake3_parallel_scratch \ diag_zeroize_hmac_sha256_finalize \ - diag_zeroize_hmac_sha3_finalize; do - FUNCTION_ASSEMBLY="$(function_assembly "$symbol")" - if ! grep -Eq "mov[bql]?[[:space:]]+\\\$0" <<<"$FUNCTION_ASSEMBLY"; then + diag_zeroize_hmac_sha3_finalize \ + diag_hkdf_sha256_derive_portable \ + diag_hkdf_sha384_derive_portable \ + diag_hkdf_sha512_derive_portable \ + diag_poly1305_block_portable_digest \ + diag_ascon_aead128_tag_portable \ + diag_aegis256_update_portable \ + diag_aes128gcm_ghash \ + diag_aes256gcm_ghash \ + diag_zeroize_mlkem_sha3_512 \ + diag_zeroize_mlkem_shake256_scalar \ + diag_zeroize_mlkem_shake256_pair \ + diag_zeroize_mlkem_shake256_quad \ + diag_zeroize_ecdsa_p256_platform_scratch; do + FUNCTION_ASSEMBLY="$(function_assembly "$symbol" <"$ASSEMBLY")" + if ! grep -Eq "$X86_ZERO_MEMORY_PATTERN" <<<"$FUNCTION_ASSEMBLY"; then echo "zeroize assembly evidence has no zero store in $symbol" >&2 exit 1 fi diff --git a/scripts/ci/changed-test-planning-test.sh b/scripts/ci/changed-test-planning-test.sh index bce6622b..05803ad7 100755 --- a/scripts/ci/changed-test-planning-test.sh +++ b/scripts/ci/changed-test-planning-test.sh @@ -103,6 +103,8 @@ make_legacy_plan() { EMPTY_SCOPE="$(make_scope empty '[]' '[]')" WORKSPACE_SCOPE="$(make_scope workspace '[]' '["--workspace"]' true true false)" CRATES_SCOPE="$(make_scope crates '["crate-a","crate-b"]' '["-p","crate-a","-p","crate-b"]' true true false)" +ACTION_WORKSPACE_SCOPE="$(jq -c '.scope_contract_version = 3 | del(.surfaces)' <<<"$WORKSPACE_SCOPE")" +ACTION_WORKSPACE_SURFACES="$(jq -c '.surfaces' <<<"$WORKSPACE_SCOPE")" EMPTY_PLAN="$(make_plan "$EMPTY_SCOPE")" WORKSPACE_PLAN="$(make_plan "$WORKSPACE_SCOPE")" CRATES_PLAN="$(make_plan "$CRATES_SCOPE")" @@ -122,7 +124,7 @@ EOF chmod +x "$planner_bin/cargo" scope_mode_for_cached_plan() ( - unset RAIL_SCOPE_JSON RAIL_SCOPE_JSON_CACHE + unset RAIL_SCOPE_JSON RAIL_SCOPE_JSON_CACHE RAIL_SURFACES_JSON export RAIL_PLAN_JSON_CACHE=$1 # shellcheck source=../lib/rail-plan.sh source "$REPO_ROOT/scripts/lib/rail-plan.sh" @@ -130,7 +132,7 @@ scope_mode_for_cached_plan() ( ) scope_mode_for_planner() ( - unset RAIL_PLAN_JSON_CACHE RAIL_SCOPE_JSON RAIL_SCOPE_JSON_CACHE + unset RAIL_PLAN_JSON_CACHE RAIL_SCOPE_JSON RAIL_SCOPE_JSON_CACHE RAIL_SURFACES_JSON export PATH="$planner_bin:$PATH" export MOCK_PLAN_OUTPUT=$1 export MOCK_PLAN_STATUS=$2 @@ -172,7 +174,7 @@ assert_eq workspace "$(scope_mode_for_cached_plan "$LEGACY_WORKSPACE_PLAN")" "va crate_output="$( ( export RAIL_PLAN_JSON_CACHE="$CRATES_PLAN" - unset RAIL_SCOPE_JSON RAIL_SCOPE_JSON_CACHE + unset RAIL_SCOPE_JSON RAIL_SCOPE_JSON_CACHE RAIL_SURFACES_JSON # shellcheck source=../lib/rail-plan.sh source "$REPO_ROOT/scripts/lib/rail-plan.sh" rail_plan_crates @@ -207,7 +209,7 @@ run_test_consumer() { : >"$command_log" if ! ( - unset RAIL_PLAN_JSON_CACHE RAIL_SCOPE_JSON RAIL_SCOPE_JSON_CACHE + unset RAIL_PLAN_JSON_CACHE RAIL_SCOPE_JSON RAIL_SCOPE_JSON_CACHE RAIL_SURFACES_JSON export HOME="$command_home" export PATH="$command_bin:$jq_dir:/usr/bin:/bin" export MOCK_LOG="$command_log" @@ -265,7 +267,7 @@ run_check_consumer() { : >"$command_log" if ! ( - unset RAIL_PLAN_JSON_CACHE RAIL_SCOPE_JSON RAIL_SCOPE_JSON_CACHE + unset RAIL_PLAN_JSON_CACHE RAIL_SCOPE_JSON RAIL_SCOPE_JSON_CACHE RAIL_SURFACES_JSON export HOME="$command_home" export PATH="$command_bin:$jq_dir:/usr/bin:/bin" export MOCK_LOG="$command_log" @@ -325,7 +327,7 @@ run_check_all_consumer() { : >"$command_log" if ! ( - unset RAIL_PLAN_JSON_CACHE RAIL_SCOPE_JSON RAIL_SCOPE_JSON_CACHE + unset RAIL_PLAN_JSON_CACHE RAIL_SCOPE_JSON RAIL_SCOPE_JSON_CACHE RAIL_SURFACES_JSON export HOME="$command_home" export PATH="$command_bin:$jq_dir:/usr/bin:/bin" export MOCK_LOG="$command_log" @@ -352,12 +354,14 @@ run_workflow_resolver() { local outcome=$2 local scope=$3 local expected=$4 + local surfaces=${5:-} local output="$TMP_ROOT/resolver-$name.outputs" : >"$output" if ! GITHUB_OUTPUT="$output" \ RAIL_PLAN_STEP_OUTCOME="$outcome" \ RAIL_SCOPE_JSON="$scope" \ + RAIL_SURFACES_JSON="$surfaces" \ bash "$REPO_ROOT/scripts/ci/resolve-rail-plan.sh"; then fail "workflow resolver failed for $name" fi @@ -370,5 +374,12 @@ run_workflow_resolver malformed-scope success '{' "$fallback_outputs" run_workflow_resolver valid-empty success "$EMPTY_SCOPE" $'valid=true\nempty=true\nbuild=false\ntest=false\ninfra=false\ncargo_graph=false' run_workflow_resolver valid-workspace success "$WORKSPACE_SCOPE" $'valid=true\nempty=false\nbuild=true\ntest=true\ninfra=false\ncargo_graph=false' run_workflow_resolver valid-crates success "$CRATES_SCOPE" $'valid=true\nempty=false\nbuild=true\ntest=true\ninfra=false\ncargo_graph=false' +run_workflow_resolver action-workspace success "$ACTION_WORKSPACE_SCOPE" \ + $'valid=true\nempty=false\nbuild=true\ntest=true\ninfra=false\ncargo_graph=false' \ + "$ACTION_WORKSPACE_SURFACES" +run_workflow_resolver action-missing-surfaces success "$ACTION_WORKSPACE_SCOPE" "$fallback_outputs" +run_workflow_resolver action-malformed-surfaces success "$ACTION_WORKSPACE_SCOPE" "$fallback_outputs" '{' +run_workflow_resolver action-mismatched-surfaces success "$WORKSPACE_SCOPE" "$fallback_outputs" \ + "$(jq -c '.test = false' <<<"$ACTION_WORKSPACE_SURFACES")" echo "Changed-test planning regression tests passed" diff --git a/scripts/ci/check-ci-ownership-test.sh b/scripts/ci/check-ci-ownership-test.sh index a9c5d863..fc269428 100755 --- a/scripts/ci/check-ci-ownership-test.sh +++ b/scripts/ci/check-ci-ownership-test.sh @@ -9,7 +9,9 @@ trap 'rm -rf "$TMP_ROOT"' EXIT make_fixture() { local fixture=$1 - mkdir -p "$fixture/.github" "$fixture/.config" "$fixture/scripts/check" "$fixture/scripts/lib" "$fixture/scripts/test" + mkdir -p "$fixture/.cargo" "$fixture/.github" "$fixture/.config" \ + "$fixture/scripts/check" "$fixture/scripts/lib" "$fixture/scripts/test" + cp "$REPO_ROOT/.cargo/config.toml" "$fixture/.cargo/config.toml" cp -R "$REPO_ROOT/.github/workflows" "$fixture/.github/workflows" cp -R "$REPO_ROOT/.github/actions" "$fixture/.github/actions" cp -R "$REPO_ROOT/.github/rulesets" "$fixture/.github/rulesets" @@ -38,6 +40,24 @@ baseline="$TMP_ROOT/baseline" make_fixture "$baseline" "$CHECKER" --root "$baseline" >/dev/null +hosted_macos="$TMP_ROOT/hosted-macos" +make_fixture "$hosted_macos" +yq eval '.jobs.hosted_macos = {"runs-on": "macos-15", "steps": [{"run": "true"}]}' -i \ + "$hosted_macos/.github/workflows/rsa.yaml" +expect_failure "$hosted_macos" "macOS testing is delegated to a hosted runner" + +apple_runner_alias="$TMP_ROOT/apple-runner-alias" +make_fixture "$apple_runner_alias" +yq eval '.jobs.apple_runner = {"uses": "./.github/workflows/_rust-job.yaml", "with": {"runner": "darwin", "operation": "check"}}' -i \ + "$apple_runner_alias/.github/workflows/rsa.yaml" +expect_failure "$apple_runner_alias" "Apple testing is delegated through a custom runner label" + +apple_rustflags="$TMP_ROOT/apple-rustflags" +make_fixture "$apple_rustflags" +printf '\n[target.aarch64-apple-darwin]\nrustflags = ["-C", "target-cpu=native"]\n' \ + >>"$apple_rustflags/.cargo/config.toml" +expect_failure "$apple_rustflags" "normal Apple builds inherit host-specific rustflags" + invalid_tool_digest="$TMP_ROOT/invalid-tool-digest" make_fixture "$invalid_tool_digest" sed -i.bak 's/ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d/not-a-digest/' \ @@ -85,6 +105,47 @@ printf '\n - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca002889 >>"$unauthenticated_rustup/.github/actions/setup-toolchain/action.yaml" expect_failure "$unauthenticated_rustup" "toolchain setup can run a network bootstrap installer" +floating_rail_action="$TMP_ROOT/floating-rail-action" +make_fixture "$floating_rail_action" +sed -i.bak \ + 's#loadingalias/cargo-rail-action@f622a3936a231fe78a772292c6892d71e8c57f9f#loadingalias/cargo-rail-action@v6#' \ + "$floating_rail_action/.github/workflows/ci.yaml" +rm -f "$floating_rail_action/.github/workflows/ci.yaml.bak" +expect_failure "$floating_rail_action" "cargo-rail-action is not commit-pinned" + +mismatched_rail_version="$TMP_ROOT/mismatched-rail-version" +make_fixture "$mismatched_rail_version" +yq eval '(.jobs."rail-plan".steps[] | select(.id == "rail") | .with.version) = "0.19.1"' -i \ + "$mismatched_rail_version/.github/workflows/ci.yaml" +expect_failure "$mismatched_rail_version" "cargo-rail-action bypasses the authenticated Cargo Rail version" + +missing_rail_checksum="$TMP_ROOT/missing-rail-checksum" +make_fixture "$missing_rail_checksum" +yq eval 'del(.jobs."rail-plan".steps[] | select(.id == "rail") | .with.checksum)' -i \ + "$missing_rail_checksum/.github/workflows/ci.yaml" +expect_failure "$missing_rail_checksum" "cargo-rail-action does not require release checksums" + +mutable_rail_base="$TMP_ROOT/mutable-rail-base" +make_fixture "$mutable_rail_base" +yq eval '(.jobs."rail-plan".steps[] | select(.id == "rail") | .with.since) = "origin/main"' -i \ + "$mutable_rail_base/.github/workflows/ci.yaml" +expect_failure "$mutable_rail_base" "cargo-rail-action plans from a mutable base" + +rail_before_authenticated_setup="$TMP_ROOT/rail-before-authenticated-setup" +make_fixture "$rail_before_authenticated_setup" +yq eval '.jobs."rail-plan".steps[1] as $setup | + .jobs."rail-plan".steps[2] as $rail | + .jobs."rail-plan".steps[1] = $rail | + .jobs."rail-plan".steps[2] = $setup' -i \ + "$rail_before_authenticated_setup/.github/workflows/ci.yaml" +expect_failure "$rail_before_authenticated_setup" "cargo-rail-action can run before authenticated installation" + +unmatched_rail_condition="$TMP_ROOT/unmatched-rail-condition" +make_fixture "$unmatched_rail_condition" +yq eval '(.jobs."rail-plan".steps[] | select(.id == "rail") | .if) = "always()"' -i \ + "$unmatched_rail_condition/.github/workflows/ci.yaml" +expect_failure "$unmatched_rail_condition" "cargo-rail-action can run without authenticated installation" + unpinned_scorecard="$TMP_ROOT/unpinned-scorecard" make_fixture "$unpinned_scorecard" sed -i.bak 's#@sha256:[0-9a-f]*#:v2.4.3#' \ diff --git a/scripts/ci/check-ci-ownership.sh b/scripts/ci/check-ci-ownership.sh index b1abbe4d..4647a55d 100755 --- a/scripts/ci/check-ci-ownership.sh +++ b/scripts/ci/check-ci-ownership.sh @@ -31,6 +31,7 @@ TOOLCHAIN_ACTION="$ACTIONS/setup-toolchain/action.yaml" SCORECARD_ACTION="$ACTIONS/scorecard/action.yaml" MANIFEST="$ROOT/.config/target-matrix.json" TOOL_ARCHIVES="$ROOT/.config/ci-tool-archives.tsv" +CARGO_CONFIG="$ROOT/.cargo/config.toml" CROSS_SCRIPT="$ROOT/scripts/ci/cross-targets.sh" NOSTD_WASM="$ROOT/scripts/ci/nostd-wasm-suite.sh" INSTALL_TOOLS="$ROOT/scripts/ci/install-tools.sh" @@ -112,6 +113,7 @@ require_file "$TOOLCHAIN_ACTION" require_file "$SCORECARD_ACTION" require_file "$MANIFEST" require_file "$TOOL_ARCHIVES" +require_file "$CARGO_CONFIG" require_file "$CROSS_SCRIPT" require_file "$NOSTD_WASM" require_file "$INSTALL_TOOLS" @@ -185,6 +187,20 @@ grep -Fq 'if [[ "$PLAN_VALID" != "true" || "$PLAN_EMPTY" != "true" ]]' "$CI" \ if grep -ERn '^[[:space:]]+(pre_script|run_script):' "$WORKFLOWS" >/dev/null; then fail "reusable workflows must not accept executable shell fragments" fi +if grep -ERin '(^|[^[:alnum:]_])(macos|darwin|apple)([^[:alnum:]_]|$)' "$WORKFLOWS" >/dev/null; then + fail "Apple platform testing must remain local and must not appear in CI workflows" +fi +if awk ' + /^\[target\./ { + apple_target = tolower($0) ~ /(apple-darwin|target_os[[:space:]]*=[[:space:]]*"macos")/ + next + } + /^\[/ { apple_target = 0 } + apple_target && /^[[:space:]]*rustflags[[:space:]]*=/ { found = 1 } + END { exit !found } +' "$CARGO_CONFIG"; then + fail "Apple targets must not receive implicit rustflags from .cargo/config.toml" +fi rust_job_calls=$(count_matches 'uses:[[:space:]]+\./\.github/workflows/_rust-job\.yaml' "$WORKFLOWS") rust_job_operations=$(count_matches '^[[:space:]]+operation:[[:space:]]+[-[:alnum:]]+[[:space:]]*$' "$WORKFLOWS") [[ "$rust_job_calls" -eq "$rust_job_operations" ]] \ @@ -220,7 +236,7 @@ fi bash -eu -o pipefail -c 'source "$1"; ci_tool_validate_manifest' _ "$TOOL_INTEGRITY" \ || fail "direct CI tool archive manifest is invalid" -if grep -ERn 'uses:[[:space:]]+(dtolnay/rust-toolchain|loadingalias/cargo-rail-action|ossf/scorecard-action)@' \ +if grep -ERn 'uses:[[:space:]]+(dtolnay/rust-toolchain|ossf/scorecard-action)@' \ "$WORKFLOWS" "$ACTIONS" >/dev/null; then fail "CI must not delegate installation to an action with an unauthenticated executable fallback" fi @@ -278,6 +294,50 @@ expected_installer_files=$(printf '%s\n' \ [[ $(yq eval '.jobs."rail-plan".steps[] | select(.id == "rail_setup") | .with."tools-mode"' "$CI") \ == "rail" ]] \ || fail "the PR planner must select the exact Cargo Rail install mode" +# The action must discover the exact package-manager install from the preceding +# step so none of its download or cargo-binstall fallbacks are reachable. +[[ $(yq eval '.jobs."rail-plan".steps[] | select(.id == "rail") | .uses' "$CI") \ + == "loadingalias/cargo-rail-action@f622a3936a231fe78a772292c6892d71e8c57f9f" ]] \ + || fail "the PR planner must use commit-pinned cargo-rail-action v6.1.0" +rail_version=$(sed -n 's/^CARGO_RAIL_VERSION=//p' "$INSTALL_TOOLS") +[[ "$rail_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] \ + || fail "the Cargo Rail installer version must be exact" +[[ $(yq eval '.jobs."rail-plan".steps[] | select(.id == "rail") | .with.version' "$CI") \ + == "$rail_version" ]] \ + || fail "cargo-rail-action must use the authenticated Cargo Rail version" +[[ $(yq eval '.jobs."rail-plan".steps[] | select(.id == "rail") | .with.checksum' "$CI") \ + == "required" ]] \ + || fail "cargo-rail-action must require release checksums" +# shellcheck disable=SC2016 # GitHub expressions are intentional literal workflow contracts. +[[ $(yq eval '.jobs."rail-plan".steps[] | select(.id == "rail") | .with.since' "$CI") \ + == '${{ github.event.pull_request.base.sha }}' ]] \ + || fail "cargo-rail-action must plan from the immutable pull-request base" +rail_setup_condition=$(yq eval '.jobs."rail-plan".steps[] | select(.id == "rail_setup") | .if' "$CI") +rail_action_condition=$(yq eval '.jobs."rail-plan".steps[] | select(.id == "rail") | .if' "$CI") +[[ "$rail_setup_condition" == "github.event_name == 'pull_request'" \ + && "$rail_action_condition" == "$rail_setup_condition" ]] \ + || fail "cargo-rail-action and its authenticated install must share the PR condition" +rail_setup_index=$(yq eval '.jobs."rail-plan".steps | to_entries | .[] | select(.value.id == "rail_setup") | .key' "$CI") +rail_action_index=$(yq eval '.jobs."rail-plan".steps | to_entries | .[] | select(.value.id == "rail") | .key' "$CI") +[[ "$rail_setup_index" =~ ^[0-9]+$ && "$rail_action_index" =~ ^[0-9]+$ \ + && "$rail_setup_index" -lt "$rail_action_index" ]] \ + || fail "the authenticated Cargo Rail install must precede cargo-rail-action" +# shellcheck disable=SC2016 # GitHub expressions are intentional literal workflow contracts. +[[ $(yq eval '.jobs."rail-plan".steps[] | select(.id == "resolve") | .env.RAIL_PLAN_STEP_OUTCOME' "$CI") \ + == '${{ steps.rail.outcome }}' ]] \ + || fail "the plan resolver must fail closed on cargo-rail-action outcome" +# shellcheck disable=SC2016 # GitHub expressions are intentional literal workflow contracts. +[[ $(yq eval '.jobs."rail-plan".steps[] | select(.id == "resolve") | .env.RAIL_SCOPE_JSON' "$CI") \ + == '${{ steps.rail.outputs.scope-json }}' ]] \ + || fail "the plan resolver must validate cargo-rail-action scope" +# shellcheck disable=SC2016 # GitHub expressions are intentional literal workflow contracts. +[[ $(yq eval '.jobs."rail-plan".steps[] | select(.id == "resolve") | .env.RAIL_SURFACES_JSON' "$CI") \ + == '${{ steps.rail.outputs.surfaces-json }}' ]] \ + || fail "the plan resolver must validate cargo-rail-action surfaces" +# shellcheck disable=SC2016 # GitHub expressions are intentional literal workflow contracts. +[[ $(yq eval '.jobs."rail-plan".steps[] | select(.name == "Check Release Intent Coverage") | .env.RAIL_BASE_REF' "$CI") \ + == '${{ steps.rail.outputs.base-ref }}' ]] \ + || fail "release intent coverage must use cargo-rail-action's resolved base" grep -Fq 'scripts/ci/setup-toolchain.sh "$TOOLCHAIN" "$TOOLCHAIN_COMPONENTS"' "$TOOLCHAIN_ACTION" \ || fail "toolchain setup must use the repository-owned rustup policy" if grep -Eq '[.]cargo/(bin|[.]crates)|[.]opam' "$SETUP_ACTION"; then @@ -303,8 +363,15 @@ grep -Fq 'go install "github.com/rhysd/actionlint/cmd/actionlint@v$ACTIONLINT_VE opam_commit=$(sed -n 's/^OPAM_REPOSITORY_COMMIT=//p' "$INSTALL_TOOLS") [[ "$opam_commit" =~ ^[0-9a-f]{40}$ ]] \ || fail "OPAM repository must use a full Git commit" +[[ $(sed -n 's/^OPAM_REPOSITORY_REMOTE=//p' "$INSTALL_TOOLS") \ + == "https://github.com/ocaml/opam-repository.git" ]] \ + || fail "OPAM repository must use the reviewed HTTPS remote" +grep -Fq 'git -C "$repository" fetch --depth=1 --no-tags' "$INSTALL_TOOLS" \ + || fail "OPAM repository must fetch only the pinned commit" grep -Fq 'actual=$(git -C "$repository" rev-parse HEAD)' "$INSTALL_TOOLS" \ || fail "OPAM metadata must be checked against its pinned commit" +grep -Fq 'status=$(git -C "$repository" status --short --untracked-files=all)' "$INSTALL_TOOLS" \ + || fail "OPAM metadata must match the pinned commit exactly" grep -Fq 'actual=$(dpkg-query -W -f=' "$INSTALL_TOOLS" \ || fail "APT packages must be validated against exact versions" grep -Fq 'ci_tool_download wasmtime' "$NOSTD_WASM" \ diff --git a/scripts/ci/check-worktree-test.sh b/scripts/ci/check-worktree-test.sh index 93df82ee..06f3b87b 100755 --- a/scripts/ci/check-worktree-test.sh +++ b/scripts/ci/check-worktree-test.sh @@ -56,7 +56,15 @@ cat >"$fixture/scripts/check/zeroize-evidence.sh" <<'EOF' exit 0 EOF -chmod +x "$fixture/scripts/ct/python.sh" "$fixture/scripts/check/zeroize-evidence.sh" +cat >"$fixture/scripts/check/rsa-asm-provenance.sh" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF + +chmod +x \ + "$fixture/scripts/ct/python.sh" \ + "$fixture/scripts/check/rsa-asm-provenance.sh" \ + "$fixture/scripts/check/zeroize-evidence.sh" cat >"$fake_bin/cargo" <<'EOF' #!/usr/bin/env bash @@ -204,7 +212,7 @@ run_recipe() { snapshot_worktree "$before" ( - unset RAIL_PLAN_JSON_CACHE RAIL_SCOPE_JSON RAIL_SCOPE_JSON_CACHE + unset RAIL_PLAN_JSON_CACHE RAIL_SCOPE_JSON RAIL_SCOPE_JSON_CACHE RAIL_SURFACES_JSON export HOME="$fake_home" export PATH="$fake_bin:$PATH" export MOCK_FORMAT_DRIFT="$format_drift" diff --git a/scripts/ci/install-tools.sh b/scripts/ci/install-tools.sh index a568bd6c..d05c6256 100755 --- a/scripts/ci/install-tools.sh +++ b/scripts/ci/install-tools.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Install exact CI tools through authenticated package-manager boundaries. +# Install CI tools through authenticated package-manager boundaries. # Usage: install-tools.sh [standard|quality|release|rail|ci|supply-chain|bench|ibm|fuzz|coverage|ct-linux|minimal|none] set -euo pipefail @@ -9,7 +9,7 @@ MODE=${1:-standard} CARGO_NEXTEST_VERSION=0.9.140 CARGO_DENY_VERSION=0.20.2 CARGO_AUDIT_VERSION=0.22.2 -CARGO_RAIL_VERSION=0.18.0 +CARGO_RAIL_VERSION=0.20.0 CARGO_SEMVER_CHECKS_VERSION=0.48.0 JUST_VERSION=1.57.0 ZIZMOR_VERSION=1.26.1 @@ -20,23 +20,23 @@ CARGO_LLVM_COV_VERSION=0.8.7 ACTIONLINT_VERSION=1.7.12 OPAM_REPOSITORY_COMMIT=49f6d620cf20ae0168cfcbeb2c33932e06cb4b74 -OPAM_REPOSITORY_URL="git+https://github.com/ocaml/opam-repository.git#$OPAM_REPOSITORY_COMMIT" +OPAM_REPOSITORY_REMOTE=https://github.com/ocaml/opam-repository.git OCAML_COMPILER_PACKAGE=ocaml-base-compiler.5.2.1 BINSEC_PACKAGE=binsec.0.11.1 BINSEC_DECODER_PACKAGE=unisim_archisec.0.0.14 BINSEC_SOLVER_PACKAGES=(bitwuzla.1.0.6 bitwuzla-cxx.0.9.0) BINSEC_APT_PACKAGES=( - build-essential=12.10ubuntu1 - git=1:2.43.0-1ubuntu7.3 - libgmp-dev=2:6.3.0+dfsg-2ubuntu6 - libmpfr-dev=4.2.1-1build1 - m4=1.4.19-4build1 - opam=2.1.5-1 - pkg-config=1.8.1-2build1 - zlib1g-dev=1:1.3.dfsg-3.1ubuntu2 + build-essential + git + libgmp-dev + libmpfr-dev + m4 + opam + pkg-config + zlib1g-dev ) -MUSL_APT_PACKAGE=musl-tools=1.2.4-2 +MUSL_APT_PACKAGE=musl-tools RSCRYPTO_TOOL_TEMP=${RUNNER_TEMP:-${TMPDIR:-/tmp}} [[ -d "$RSCRYPTO_TOOL_TEMP" ]] || { @@ -153,19 +153,32 @@ require_ubuntu_24_04() { os_id=$(sed -n 's/^ID=//p' /etc/os-release | tr -d '"') os_version=$(sed -n 's/^VERSION_ID=//p' /etc/os-release | tr -d '"') [[ "$os_id" == ubuntu && "$os_version" == 24.04 ]] \ - || fail "exact APT pins support Ubuntu 24.04, found $os_id $os_version" + || fail "APT package installation supports Ubuntu 24.04, found $os_id $os_version" } -apt_install_exact() { +apt_install_authenticated_candidates() { require_ubuntu_24_04 command -v apt-get >/dev/null 2>&1 || fail "apt-get is required" + command -v apt-cache >/dev/null 2>&1 || fail "apt-cache is required" command -v dpkg-query >/dev/null 2>&1 || fail "dpkg-query is required" - sudo apt-get update - sudo apt-get install -y --no-install-recommends --allow-downgrades "$@" + sudo apt-get --no-allow-insecure-repositories --error-on=any update - local specification package expected actual - for specification in "$@"; do + local -a specifications=() + local package candidate + for package in "$@"; do + candidate=$(LC_ALL=C apt-cache policy "$package" | sed -n 's/^[[:space:]]*Candidate:[[:space:]]*//p') + [[ -n "$candidate" && "$candidate" != "(none)" ]] \ + || fail "signed APT metadata has no candidate for $package" + specifications+=("$package=$candidate") + done + + sudo apt-get install -y --no-install-recommends \ + --no-allow-unauthenticated --no-allow-downgrades --no-remove \ + "${specifications[@]}" + + local specification expected actual + for specification in "${specifications[@]}"; do package=${specification%%=*} expected=${specification#*=} actual=$(dpkg-query -W -f='${Version}' "$package") \ @@ -182,18 +195,30 @@ install_binsec_system_packages() { if [[ "${BINSEC_SYSTEM_PACKAGES_READY:-}" == 1 ]]; then return 0 fi - apt_install_exact "${BINSEC_APT_PACKAGES[@]}" + apt_install_authenticated_candidates "${BINSEC_APT_PACKAGES[@]}" BINSEC_SYSTEM_PACKAGES_READY=1 } verify_opam_repository() { - local repository="$OPAMROOT/repo/default" - [[ -d "$repository/.git" ]] \ - || fail "OPAM repository is not the pinned Git checkout" - local actual - actual=$(git -C "$repository" rev-parse HEAD) + local repository=$1 + local actual status + actual=$(git -C "$repository" rev-parse HEAD) \ + || fail "unable to read the OPAM repository commit" [[ "$actual" == "$OPAM_REPOSITORY_COMMIT" ]] \ || fail "OPAM repository is $actual, expected $OPAM_REPOSITORY_COMMIT" + status=$(git -C "$repository" status --short --untracked-files=all) \ + || fail "unable to verify the OPAM repository worktree" + [[ -z "$status" ]] \ + || fail "OPAM repository differs from its pinned commit" +} + +checkout_opam_repository() { + local repository=$1 + git init --quiet "$repository" + git -C "$repository" fetch --depth=1 --no-tags \ + "$OPAM_REPOSITORY_REMOTE" "$OPAM_REPOSITORY_COMMIT" + git -C "$repository" checkout --quiet --detach FETCH_HEAD + verify_opam_repository "$repository" } opam_package_is_installed() { @@ -219,9 +244,11 @@ install_binsec() { export OPAMROOT="$RSCRYPTO_TOOL_ROOT/opam" export OPAMSWITCH=rscrypto-ct + local repository="$RSCRYPTO_TOOL_ROOT/opam-repository" + checkout_opam_repository "$repository" opam init --bare --disable-sandboxing --no-setup --no-opamrc -y \ - default "$OPAM_REPOSITORY_URL" - verify_opam_repository + default "$repository" + verify_opam_repository "$repository" opam switch create "$OPAMSWITCH" "$OCAML_COMPILER_PACKAGE" \ --repositories=default -y @@ -233,6 +260,7 @@ install_binsec() { ) opam install --switch="$OPAMSWITCH" "${required_packages[@]}" -y opam reinstall --switch="$OPAMSWITCH" "$BINSEC_PACKAGE" -y + verify_opam_repository "$repository" verify_opam_packages local switch_bin @@ -253,7 +281,7 @@ install_binsec() { } install_ct_linux_packages() { - apt_install_exact "${BINSEC_APT_PACKAGES[@]}" "$MUSL_APT_PACKAGE" + apt_install_authenticated_candidates "${BINSEC_APT_PACKAGES[@]}" "$MUSL_APT_PACKAGE" BINSEC_SYSTEM_PACKAGES_READY=1 } diff --git a/scripts/ci/pre-push-test.sh b/scripts/ci/pre-push-test.sh index 0952e01d..1b3edb4f 100755 --- a/scripts/ci/pre-push-test.sh +++ b/scripts/ci/pre-push-test.sh @@ -50,6 +50,7 @@ if ( RAIL_PLAN_JSON_CACHE="$plan" \ RAIL_SCOPE_JSON='' \ RAIL_SCOPE_JSON_CACHE='' \ + RAIL_SURFACES_JSON='' \ scripts/ci/pre-push.sh --light ) >"$normal_output" 2>&1; then echo "ordinary pushes must fail when release intent coverage fails" >&2 @@ -67,7 +68,7 @@ run_pre_push_case() { : >"$mock_log" if ! ( - unset RAIL_PLAN_JSON_CACHE RAIL_SCOPE_JSON RAIL_SCOPE_JSON_CACHE + unset RAIL_PLAN_JSON_CACHE RAIL_SCOPE_JSON RAIL_SCOPE_JSON_CACHE RAIL_SURFACES_JSON if [[ -n "$cached_plan" ]]; then export RAIL_PLAN_JSON_CACHE="$cached_plan" fi @@ -117,6 +118,7 @@ if ! ( RAIL_PLAN_JSON_CACHE="$installer_plan" \ RAIL_SCOPE_JSON='' \ RAIL_SCOPE_JSON_CACHE='' \ + RAIL_SURFACES_JSON='' \ scripts/ci/pre-push.sh --light ) >/dev/null 2>&1; then echo "pre-push installer-integrity routing failed" >&2 diff --git a/scripts/ci/pre-push.sh b/scripts/ci/pre-push.sh index 2d72424e..958eda7e 100755 --- a/scripts/ci/pre-push.sh +++ b/scripts/ci/pre-push.sh @@ -71,6 +71,7 @@ TASK_NAMES=() TASK_PIDS=() TASK_LOGS=() +unset RAIL_SURFACES_JSON RAIL_READY=false if RAIL_PLAN_JSON_CACHE="$(rail_plan_json)" \ && rail_plan_is_valid "$RAIL_PLAN_JSON_CACHE"; then diff --git a/scripts/ci/release-identity-test.sh b/scripts/ci/release-identity-test.sh index 692f0cd0..2a32782d 100755 --- a/scripts/ci/release-identity-test.sh +++ b/scripts/ci/release-identity-test.sh @@ -96,7 +96,7 @@ jq -e --arg commit "$commit" ' and (.release.tag_object | test("^[0-9a-f]{40}$")) and .release.git_commit == $commit and (.release.git_tree | test("^[0-9a-f]{40}$")) - and .toolchain.channel == "nightly-2026-04-27" + and .toolchain.channel == "nightly-2026-07-17" and (.toolchain.manifest.sha256 | test("^[0-9a-f]{64}$")) and .evidence.git_commit == $commit and .evidence.mode == "exact_commit" diff --git a/scripts/ci/run-rust-job-test.sh b/scripts/ci/run-rust-job-test.sh index f0f6d233..dfcb1a16 100755 --- a/scripts/ci/run-rust-job-test.sh +++ b/scripts/ci/run-rust-job-test.sh @@ -20,13 +20,15 @@ expect_failure() { FIXTURE="$TMP_ROOT/repo" CAPTURE="$TMP_ROOT/capture" BIN="$TMP_ROOT/bin" -mkdir -p "$FIXTURE/scripts/ci" "$FIXTURE/scripts/ct" "$CAPTURE" "$BIN" +AMX_BIN="$TMP_ROOT/amx-bin" +mkdir -p "$FIXTURE/scripts/ci" "$FIXTURE/scripts/ct" "$CAPTURE" "$BIN" "$AMX_BIN" cp "$DISPATCHER" "$FIXTURE/scripts/ci/run-rust-job.sh" cat >"$BIN/just" <<'EOF' #!/usr/bin/env bash set -euo pipefail printf '%s\n' "$@" >"$RSCRYPTO_CI_CAPTURE_DIR/just.args" +exit "${RSCRYPTO_MOCK_JUST_STATUS:-0}" EOF for command in uname lscpu sed rustc cargo; do @@ -67,12 +69,109 @@ chmod +x \ "$FIXTURE/scripts/ct/full.py" \ "$FIXTURE/scripts/ct/python.sh" +cat >"$AMX_BIN/uname" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +case "${1:-}" in + -s) echo Linux ;; + -m) echo x86_64 ;; + *) echo "Linux AMX fixture" ;; +esac +EOF + +cat >"$AMX_BIN/lscpu" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +echo "Architecture: x86_64" +EOF + +cat >"$AMX_BIN/rustc" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +echo "host: x86_64-unknown-linux-gnu" +EOF + +cat >"$AMX_BIN/sed" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +if [[ "${*: -1}" == /proc/cpuinfo ]]; then + if [[ "$*" == *'s/^flags'* ]]; then + echo "amx_tile" + else + echo "flags : amx_tile" + fi +else + /usr/bin/sed "$@" +fi +EOF + +cat >"$AMX_BIN/cargo" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf 'RUSTFLAGS=%s RSCRYPTO_REQUIRE_AMX=%s :: %s\n' \ + "${RUSTFLAGS:-}" "${RSCRYPTO_REQUIRE_AMX:-}" "$*" \ + >>"$RSCRYPTO_CI_CAPTURE_DIR/amx-cargo.args" + +if [[ " $* " != *" --list "* ]]; then + exit 0 +fi + +case "${RSCRYPTO_MOCK_AMX_LIST:-none}" in + all) + if [[ " $* " == *" --test platform_amx_permission "* ]]; then + echo "linux_x86_64_amx_permission_and_cache_are_process_scoped: test" + elif [[ " $* " == *" --lib "* ]]; then + echo "platform::detect::tests::no_std_linux_x86_64_masks_compile_time_amx_without_a_permission_probe: test" + fi + ;; + integration-only) + if [[ " $* " == *" --test platform_amx_permission "* ]]; then + echo "linux_x86_64_amx_permission_and_cache_are_process_scoped: test" + fi + ;; + none) ;; + *) exit 91 ;; +esac +EOF + +chmod +x "$AMX_BIN/uname" "$AMX_BIN/lscpu" "$AMX_BIN/rustc" "$AMX_BIN/sed" "$AMX_BIN/cargo" + TEST_PATH="$BIN:$PATH" RUNNER=(env PATH="$TEST_PATH" RSCRYPTO_CI_CAPTURE_DIR="$CAPTURE" bash "$FIXTURE/scripts/ci/run-rust-job.sh") RSCRYPTO_CI_OPERATION=quality "${RUNNER[@]}" [[ $(<"$CAPTURE/just.args") == "ci-check" ]] || fail "quality selected the wrong command" +mkdir -p \ + "$FIXTURE/fuzz/corpus/hash_cshake256" \ + "$FIXTURE/fuzz/artifacts/hash_cshake256" \ + "$FIXTURE/fuzz-packages/hash-sha3/corpus/hash_cshake256" \ + "$FIXTURE/fuzz-packages/hash-sha3/artifacts/hash_cshake256" +printf '%s' full-corpus >"$FIXTURE/fuzz/corpus/hash_cshake256/seed" +printf '%s' scoped-corpus >"$FIXTURE/fuzz-packages/hash-sha3/corpus/hash_cshake256/seed" +printf '%s' full-crash >"$FIXTURE/fuzz/artifacts/hash_cshake256/crash-fixture" +printf '%s' scoped-crash >"$FIXTURE/fuzz-packages/hash-sha3/artifacts/hash_cshake256/crash-fixture" + +fuzz_status=0 +env \ + PATH="$TEST_PATH" \ + RSCRYPTO_CI_CAPTURE_DIR="$CAPTURE" \ + RSCRYPTO_CI_OPERATION=fuzz \ + RSCRYPTO_MOCK_JUST_STATUS=23 \ + bash "$FIXTURE/scripts/ci/run-rust-job.sh" >/dev/null 2>&1 \ + || fuzz_status=$? +[[ "$fuzz_status" -eq 23 ]] || fail "fuzz operation did not preserve the fuzz command's failure status" +[[ -f "$FIXTURE/fuzz-output/corpus.tar.gz" ]] || fail "fuzz failure did not produce a corpus archive" +tar -tzf "$FIXTURE/fuzz-output/corpus.tar.gz" >"$CAPTURE/fuzz-archive.entries" +grep -Fxq 'fuzz/corpus/hash_cshake256/seed' "$CAPTURE/fuzz-archive.entries" \ + || fail "fuzz failure archive omitted the full-workspace corpus" +grep -Fxq 'fuzz-packages/hash-sha3/corpus/hash_cshake256/seed' "$CAPTURE/fuzz-archive.entries" \ + || fail "fuzz failure archive omitted the scoped corpus" +[[ -f "$FIXTURE/fuzz/artifacts/hash_cshake256/crash-fixture" ]] \ + || fail "fuzz failure removed the full-workspace crash artifact" +[[ -f "$FIXTURE/fuzz-packages/hash-sha3/artifacts/hash_cshake256/crash-fixture" ]] \ + || fail "fuzz failure removed the scoped crash artifact" + sentinel="$TMP_ROOT/injected" # shellcheck disable=SC2016 # Command substitution is an intentional literal injection payload. shell_payload='$(touch "'"$sentinel"'"); echo injected; #' @@ -136,6 +235,42 @@ expect_failure env \ bash "$FIXTURE/scripts/ci/run-rust-job.sh" [[ ! -e "$sentinel" ]] || fail "numeric input was evaluated as shell code" +AMX_PATH="$AMX_BIN:$PATH" +expect_failure env \ + PATH="$AMX_PATH" \ + RSCRYPTO_CI_CAPTURE_DIR="$CAPTURE" \ + RSCRYPTO_CI_OPERATION=platform-amx \ + RSCRYPTO_CI_RUNNER=intel-spr \ + RSCRYPTO_MOCK_AMX_LIST=none \ + bash "$FIXTURE/scripts/ci/run-rust-job.sh" + +expect_failure env \ + PATH="$AMX_PATH" \ + RSCRYPTO_CI_CAPTURE_DIR="$CAPTURE" \ + RSCRYPTO_CI_OPERATION=platform-amx \ + RSCRYPTO_CI_RUNNER=intel-spr \ + RSCRYPTO_MOCK_AMX_LIST=integration-only \ + bash "$FIXTURE/scripts/ci/run-rust-job.sh" + +: >"$CAPTURE/amx-cargo.args" +env \ + PATH="$AMX_PATH" \ + RSCRYPTO_CI_CAPTURE_DIR="$CAPTURE" \ + RSCRYPTO_CI_OPERATION=platform-amx \ + RSCRYPTO_CI_RUNNER=intel-spr \ + RSCRYPTO_MOCK_AMX_LIST=all \ + bash "$FIXTURE/scripts/ci/run-rust-job.sh" >/dev/null +[[ "$(wc -l <"$CAPTURE/amx-cargo.args" | tr -d ' ')" == 4 ]] \ + || fail "AMX operation did not list and run both exact tests" +grep -Fq \ + 'RSCRYPTO_REQUIRE_AMX=1 :: test --locked --test platform_amx_permission -- --list' \ + "$CAPTURE/amx-cargo.args" \ + || fail "AMX integration test existence was not checked under the required permission contract" +grep -Fq \ + 'RUSTFLAGS=-A unstable-features -C target-feature=+amx-tile,+amx-bf16,+amx-int8' \ + "$CAPTURE/amx-cargo.args" \ + || fail "AMX no_std test existence was not checked with forced AMX target features" + if grep -En '(^|[[:space:]])eval[[:space:]]|(^|[[:space:]])(bash|sh)[[:space:]]+-c|<<<' "$DISPATCHER" >/dev/null; then fail "dispatcher contains a dynamic shell interpreter" fi diff --git a/scripts/ci/run-rust-job.sh b/scripts/ci/run-rust-job.sh index 595a0c03..ce497e89 100755 --- a/scripts/ci/run-rust-job.sh +++ b/scripts/ci/run-rust-job.sh @@ -47,6 +47,18 @@ require_commit_sha() { || die "base_sha must be a full commit ID" } +assert_single_libtest() { + local test_name=$1 + shift + + local listing count + if ! listing=$("$@" --list); then + die "unable to list the test harness containing $test_name" + fi + count=$(printf '%s\n' "$listing" | awk -v expected="$test_name: test" '$0 == expected { count++ } END { print count + 0 }') + [[ "$count" -eq 1 ]] || die "expected exactly one libtest named $test_name; found $count" +} + host_diagnostics() { local cpuinfo_lines=$1 uname -a @@ -122,6 +134,40 @@ run_native_riscv() { bash scripts/test/test.sh --all } +run_platform_amx() { + local runner=${RSCRYPTO_CI_RUNNER:-} + [[ "$runner" == *"intel-spr"* ]] || die "AMX permission evidence requires the intel-spr runner" + [[ "$(uname -s)" == Linux ]] || die "AMX permission evidence requires Linux" + [[ "$(uname -m)" == x86_64 ]] || die "AMX permission evidence requires x86-64" + [[ "$(rustc -vV | sed -n 's/^host: //p')" == x86_64-unknown-linux-gnu ]] \ + || die "AMX permission evidence requires the x86_64-unknown-linux-gnu Rust host" + + host_diagnostics 80 + local flags + flags=$(sed -n 's/^flags[[:space:]]*: //p' /proc/cpuinfo | head -n 1) + [[ " $flags " == *" amx_tile "* ]] || die "intel-spr runner does not expose AMX-TILE" + + RSCRYPTO_REQUIRE_AMX=1 \ + assert_single_libtest \ + linux_x86_64_amx_permission_and_cache_are_process_scoped \ + cargo test --locked --test platform_amx_permission -- + RSCRYPTO_REQUIRE_AMX=1 \ + cargo test --locked --test platform_amx_permission \ + linux_x86_64_amx_permission_and_cache_are_process_scoped -- --exact --nocapture + + # NIGHTLY: Rust target-feature names for AMX remain unstable. This lane + # deliberately forces them so the no_std permission gate is executable. + local amx_rustflags="-A unstable-features -C target-feature=+amx-tile,+amx-bf16,+amx-int8" + RUSTFLAGS="$amx_rustflags" \ + assert_single_libtest \ + platform::detect::tests::no_std_linux_x86_64_masks_compile_time_amx_without_a_permission_probe \ + cargo test --locked --no-default-features --lib -- + RUSTFLAGS="$amx_rustflags" \ + cargo test --locked --no-default-features --lib \ + platform::detect::tests::no_std_linux_x86_64_masks_compile_time_amx_without_a_permission_probe \ + -- --exact --nocapture +} + run_cross_targets() { bash scripts/ci/cross-targets.sh deep } @@ -159,7 +205,8 @@ run_miri() { run_fuzz() { export RSCRYPTO_FUZZ_DURATION_SECS=60 - just test-fuzz --all + local fuzz_status=0 + just test-fuzz --all || fuzz_status=$? rm -rf -- fuzz-output mkdir -p fuzz-output @@ -174,6 +221,8 @@ run_fuzz() { else tar -czf fuzz-output/corpus.tar.gz "${corpus_dirs[@]}" fi + + return "$fuzz_status" } run_fuzz_asan() { @@ -344,6 +393,58 @@ run_rsa_leakage() { } 2>&1 | tee "ci-evidence/rsa-leakage-$target.log" } +run_rsa_linux_x86_64_asm() { + mkdir -p ci-evidence + { + uname -a + [[ "$(uname -s)" == Linux ]] || die "RSA x86-64 assembly evidence requires Linux" + [[ "$(uname -m)" == x86_64 ]] || die "RSA x86-64 assembly evidence requires x86-64" + [[ "$(rustc -vV | sed -n 's/^host: //p')" == x86_64-unknown-linux-gnu ]] \ + || die "RSA x86-64 assembly evidence requires the x86_64-unknown-linux-gnu Rust host" + + local flags + flags=$(sed -n 's/^flags[[:space:]]*: //p' /proc/cpuinfo | head -n 1) + [[ " $flags " == *" bmi2 "* ]] || die "RSA x86-64 assembly evidence requires BMI2" + [[ " $flags " == *" adx "* ]] || die "RSA x86-64 assembly evidence requires ADX" + lscpu + + assert_single_libtest \ + auth::rsa::tests::x86_64_linux_rsa_montgomery_asm_matches_portable_across_supported_widths \ + cargo test --locked --features rsa,diag,getrandom --lib -- + cargo test --locked --features rsa,diag,getrandom --lib \ + auth::rsa::tests::x86_64_linux_rsa_montgomery_asm_matches_portable_across_supported_widths \ + -- --exact --nocapture + assert_single_libtest \ + auth::rsa::tests::x86_64_linux_rsa_montgomery_asm_matches_portable_across_supported_widths \ + cargo test --locked --release --features rsa,diag,getrandom --lib -- + cargo test --locked --release --features rsa,diag,getrandom --lib \ + auth::rsa::tests::x86_64_linux_rsa_montgomery_asm_matches_portable_across_supported_widths \ + -- --exact --nocapture + + local build_output binary + build_output=$(cargo test --locked --release --features rsa,diag \ + --test rsa_public_key --no-run --message-format=json) + binary=$(printf '%s\n' "$build_output" \ + | sed -n 's/.*"executable":"\([^"]*rsa_public_key-[^"]*\)".*/\1/p' \ + | tail -n 1) + [[ -n "$binary" && -x "$binary" ]] \ + || die "unable to resolve the optimized rsa_public_key test binary" + printf 'Optimized RSA test binary: %s\n' "$binary" + assert_single_libtest public_operation_montgomery_candidates_match_current_path "$binary" + "$binary" public_operation_montgomery_candidates_match_current_path --exact --nocapture + + local binary_description binary_symbols + binary_description=$(file "$binary") || die "unable to inspect the optimized rsa_public_key test binary" + [[ "$binary_description" == *"ELF 64-bit LSB pie executable, x86-64"* ]] \ + || die "optimized rsa_public_key test binary is not x86-64 ELF" + binary_symbols=$(nm "$binary") || die "unable to read the optimized rsa_public_key symbol table" + [[ "$binary_symbols" == *"rscrypto_rsa_bn_mulx4x_mont_x86_64_elf"* ]] \ + || die "optimized rsa_public_key test binary lacks the x86-64 Montgomery multiply" + [[ "$binary_symbols" == *"rscrypto_rsa_bn_sqr8x_mont_x86_64_elf"* ]] \ + || die "optimized rsa_public_key test binary lacks the x86-64 Montgomery square" + } 2>&1 | tee ci-evidence/rsa-linux-x86_64-asm.log +} + main() { if [[ $# -ne 0 ]]; then die "usage: scripts/ci/run-rust-job.sh" @@ -358,6 +459,7 @@ main() { native) run_native ;; native-ibm) run_native_ibm ;; native-riscv) run_native_riscv ;; + platform-amx) run_platform_amx ;; cross-targets) run_cross_targets ;; supply-chain) run_supply_chain ;; dependabot-smoke) run_dependabot_smoke ;; @@ -369,6 +471,7 @@ main() { constant-time) run_constant_time ;; rsa-miri) run_rsa_miri ;; rsa-leakage) run_rsa_leakage ;; + rsa-linux-x64-asm) run_rsa_linux_x86_64_asm ;; *) die "unsupported operation: $operation" ;; esac } diff --git a/scripts/ci/tool-integrity-test.sh b/scripts/ci/tool-integrity-test.sh index 50d521c2..b9dba453 100755 --- a/scripts/ci/tool-integrity-test.sh +++ b/scripts/ci/tool-integrity-test.sh @@ -350,7 +350,7 @@ SH cat >"$package_bin/rustc" <<'SH' #!/usr/bin/env bash -printf 'rustc 1.97.0-nightly\ncommit-date: 2026-04-26\n' +printf 'rustc 1.99.0-nightly\ncommit-date: 2026-07-16\n' SH cp "$direct_bin/uname" "$package_bin/uname" @@ -382,18 +382,37 @@ set -euo pipefail printf 'apt-get %s\n' "$*" >>"$MOCK_PACKAGE_LOG" SH +cat >"$package_bin/apt-cache" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +[[ "$1" == policy && $# == 2 ]] +case "$2" in + build-essential) candidate=12.10ubuntu1 ;; + git) candidate=1:2.54.0-0ppa1~ubuntu24.04.1 ;; + libgmp-dev) candidate=2:6.3.0+dfsg-2ubuntu6.1 ;; + libmpfr-dev) candidate=4.2.1-1build1.1 ;; + m4) candidate=1.4.19-4build1 ;; + opam) candidate=2.1.5-1 ;; + pkg-config) candidate=1.8.1-2build1 ;; + zlib1g-dev) candidate=1:1.3.dfsg-3.1ubuntu2.1 ;; + musl-tools) candidate=1.2.4-2 ;; + *) candidate='(none)' ;; +esac +printf '%s:\n Candidate: %s\n' "$2" "$candidate" +SH + cat >"$package_bin/dpkg-query" <<'SH' #!/usr/bin/env bash set -euo pipefail case "${*: -1}" in build-essential) printf '12.10ubuntu1' ;; - git) printf '1:2.43.0-1ubuntu7.3' ;; - libgmp-dev) printf '2:6.3.0+dfsg-2ubuntu6' ;; - libmpfr-dev) printf '4.2.1-1build1' ;; + git) printf '1:2.54.0-0ppa1~ubuntu24.04.1' ;; + libgmp-dev) printf '2:6.3.0+dfsg-2ubuntu6.1' ;; + libmpfr-dev) printf '4.2.1-1build1.1' ;; m4) printf '1.4.19-4build1' ;; opam) printf '2.1.5-1' ;; pkg-config) printf '1.8.1-2build1' ;; - zlib1g-dev) printf '1:1.3.dfsg-3.1ubuntu2' ;; + zlib1g-dev) printf '1:1.3.dfsg-3.1ubuntu2.1' ;; musl-tools) printf '1.2.4-2' ;; *) exit 96 ;; esac @@ -403,8 +422,39 @@ cat >"$package_bin/git" <<'SH' #!/usr/bin/env bash set -euo pipefail printf 'git %s\n' "$*" >>"$MOCK_PACKAGE_LOG" -[[ "$1" == -C && "$3" == rev-parse && "$4" == HEAD ]] -printf '49f6d620cf20ae0168cfcbeb2c33932e06cb4b74\n' +case "$1" in + init) + [[ "$2" == --quiet ]] + mkdir -p "$3/.git" + ;; + -C) + repository=$2 + shift 2 + case "$1" in + fetch) + [[ "$2" == --depth=1 && "$3" == --no-tags ]] + [[ "$4" == https://github.com/ocaml/opam-repository.git ]] + [[ "$5" == 49f6d620cf20ae0168cfcbeb2c33932e06cb4b74 ]] + ;; + checkout) + [[ "$2" == --quiet && "$3" == --detach && "$4" == FETCH_HEAD ]] + ;; + rev-parse) + [[ "$2" == HEAD ]] + printf '%s\n' "${MOCK_GIT_HEAD:-49f6d620cf20ae0168cfcbeb2c33932e06cb4b74}" + ;; + status) + [[ "$2" == --short && "$3" == --untracked-files=all ]] + if [[ "${MOCK_GIT_DIRTY:-0}" == 1 ]]; then + printf ' M packages/binsec/binsec.0.11.1/opam\n' + fi + ;; + *) exit 95 ;; + esac + [[ -d "$repository/.git" ]] + ;; + *) exit 94 ;; +esac SH cat >"$package_bin/opam" <<'SH' @@ -412,9 +462,7 @@ cat >"$package_bin/opam" <<'SH' set -euo pipefail printf 'opamroot=%s opam %s\n' "$OPAMROOT" "$*" >>"$MOCK_PACKAGE_LOG" case "$1" in - init) - mkdir -p "$OPAMROOT/repo/default/.git" - ;; + init) ;; switch) [[ "$2" == create ]] ;; @@ -442,7 +490,7 @@ EOF *) exit 97 ;; esac SH -chmod +x "$package_bin/sudo" "$package_bin/apt-get" \ +chmod +x "$package_bin/sudo" "$package_bin/apt-get" "$package_bin/apt-cache" \ "$package_bin/dpkg-query" "$package_bin/git" "$package_bin/opam" ct_home="$TMP_ROOT/ct-home" @@ -465,19 +513,52 @@ HOME="$ct_home" \ MOCK_CARGO_STATE="$ct_state" \ "$ct_installer" ct-linux >/dev/null grep -Fq \ - 'apt-get install -y --no-install-recommends --allow-downgrades build-essential=12.10ubuntu1 git=1:2.43.0-1ubuntu7.3' \ - "$ct_log" || fail "ct-linux did not select exact Ubuntu package versions" + 'apt-get --no-allow-insecure-repositories --error-on=any update' \ + "$ct_log" || fail "ct-linux did not require authenticated APT metadata" grep -Fq \ - 'opam init --bare --disable-sandboxing --no-setup --no-opamrc -y default git+https://github.com/ocaml/opam-repository.git#49f6d620cf20ae0168cfcbeb2c33932e06cb4b74' \ - "$ct_log" || fail "ct-linux did not select the commit-pinned OPAM repository" + 'apt-get install -y --no-install-recommends --no-allow-unauthenticated --no-allow-downgrades --no-remove build-essential=12.10ubuntu1 git=1:2.54.0-0ppa1~ubuntu24.04.1 libgmp-dev=2:6.3.0+dfsg-2ubuntu6.1' \ + "$ct_log" || fail "ct-linux did not install signed Ubuntu package candidates exactly" +if grep -Fq -- '--allow-downgrades' "$ct_log"; then + fail "ct-linux permits APT package downgrades" +fi +grep -Eq \ + '^git -C .*/ct-temp/rscrypto-ci-tools\.[^/]+/opam-repository fetch --depth=1 --no-tags https://github.com/ocaml/opam-repository\.git 49f6d620cf20ae0168cfcbeb2c33932e06cb4b74$' \ + "$ct_log" || fail "ct-linux did not fetch the exact OPAM repository commit" +grep -Eq \ + '^opamroot=.*/ct-temp/rscrypto-ci-tools\.[^/]+/opam opam init --bare --disable-sandboxing --no-setup --no-opamrc -y default .*/ct-temp/rscrypto-ci-tools\.[^/]+/opam-repository$' \ + "$ct_log" || fail "ct-linux did not initialize OPAM from the verified checkout" grep -Eq '^opamroot=.*/ct-temp/rscrypto-ci-tools\.[^/]+/opam opam switch create rscrypto-ct ocaml-base-compiler\.5\.2\.1 ' \ "$ct_log" || fail "ct-linux did not use a fresh exact OPAM switch" +bad_commit_temp="$TMP_ROOT/ct-bad-commit" +mkdir -p "$bad_commit_temp" +if HOME="$ct_home" \ + RUNNER_TEMP="$bad_commit_temp" \ + PATH="$package_bin:$PATH" \ + MOCK_PACKAGE_LOG="$TMP_ROOT/ct-bad-commit.log" \ + MOCK_CARGO_STATE="$ct_state" \ + MOCK_GIT_HEAD=0000000000000000000000000000000000000000 \ + "$ct_installer" ct-linux >/dev/null 2>&1; then + fail "ct-linux accepted the wrong OPAM repository commit" +fi + +dirty_repository_temp="$TMP_ROOT/ct-dirty-repository" +mkdir -p "$dirty_repository_temp" +if HOME="$ct_home" \ + RUNNER_TEMP="$dirty_repository_temp" \ + PATH="$package_bin:$PATH" \ + MOCK_PACKAGE_LOG="$TMP_ROOT/ct-dirty-repository.log" \ + MOCK_CARGO_STATE="$ct_state" \ + MOCK_GIT_DIRTY=1 \ + "$ct_installer" ct-linux >/dev/null 2>&1; then + fail "ct-linux accepted modified OPAM repository metadata" +fi + for contract in \ 'cargo-nextest =0.9.140' \ 'cargo-deny =0.20.2' \ 'cargo-audit =0.22.2' \ - 'cargo-rail =0.18.0' \ + 'cargo-rail =0.20.0' \ 'cargo-semver-checks =0.48.0' \ 'just =1.57.0' \ 'zizmor =1.26.1' \ @@ -537,11 +618,11 @@ esac : >"$package_log" MOCK_PACKAGE_LOG="$package_log" PATH="$package_bin:$PATH" \ "$REPO_ROOT/scripts/ci/setup-toolchain.sh" \ - nightly-2026-04-27 'clippy, rustfmt' >/dev/null + nightly-2026-07-17 'clippy, rustfmt' >/dev/null grep -Fq \ - 'rustup toolchain install nightly-2026-04-27 --profile minimal --no-self-update --component clippy --component rustfmt' \ + 'rustup toolchain install nightly-2026-07-17 --profile minimal --no-self-update --component clippy --component rustfmt' \ "$package_log" || fail "rustup toolchain command was not exact" -grep -Fq 'rustup default nightly-2026-04-27' "$package_log" \ +grep -Fq 'rustup default nightly-2026-07-17' "$package_log" \ || fail "rustup did not select the exact toolchain" if MOCK_PACKAGE_LOG="$package_log" PATH="$package_bin:$PATH" \ "$REPO_ROOT/scripts/ci/setup-toolchain.sh" nightly clippy >/dev/null 2>&1; then diff --git a/scripts/ct/asm_heuristics.py b/scripts/ct/asm_heuristics.py index 2ec4f2b9..07c6510a 100755 --- a/scripts/ct/asm_heuristics.py +++ b/scripts/ct/asm_heuristics.py @@ -309,15 +309,27 @@ def public_operand_rules(root: Path) -> list[dict[str, Any]]: def apply_public_operand_rules(findings: list[dict[str, Any]], configured: list[dict[str, Any]]) -> list[str]: - required = {"primitive", "root", "symbol", "kind", "max_count", "source", "rationale"} + required = {"primitives", "roots", "symbol", "kind", "max_count", "source", "rationale"} errors: list[str] = [] match_counts = [0] * len(configured) for index, rule in enumerate(configured): if set(rule) != required: errors.append(f"asm_public_operand[{index}] must contain exactly: {', '.join(sorted(required))}") continue - if any(not isinstance(rule.get(field), str) or not rule.get(field, "").strip() for field in required - {"max_count"}): + if any( + not isinstance(rule.get(field), str) or not rule.get(field, "").strip() + for field in required - {"max_count", "primitives", "roots"} + ): errors.append(f"asm_public_operand[{index}] string fields must be non-empty") + for field in ("primitives", "roots"): + values = rule.get(field) + if ( + not isinstance(values, list) + or not values + or any(not isinstance(value, str) or not value.strip() for value in values) + or len(values) != len(set(values)) + ): + errors.append(f"asm_public_operand[{index}] {field} must be a non-empty list of unique strings") max_count = rule.get("max_count") if isinstance(max_count, bool) or not isinstance(max_count, int) or max_count <= 0: errors.append(f"asm_public_operand[{index}] max_count must be a positive integer") @@ -331,8 +343,8 @@ def apply_public_operand_rules(findings: list[dict[str, Any]], configured: list[ for index, rule in enumerate(configured) if rule["symbol"] == finding["symbol"] and rule["kind"] == finding["kind"] - and rule["primitive"] in finding.get("primitive_ids", []) - and rule["root"] in finding.get("roots", []) + and set(rule["primitives"]).issubset(finding.get("primitive_ids", [])) + and set(finding.get("roots", [])).issubset(rule["roots"]) ] if len(matches) > 1: errors.append(f"multiple asm_public_operand rules match {finding['locator']}") @@ -345,8 +357,8 @@ def apply_public_operand_rules(findings: list[dict[str, Any]], configured: list[ finding["operand_class"] = "public" finding["disposition"] = "accepted" finding["public_classification"] = { - "primitive": rule["primitive"], - "root": rule["root"], + "primitives": rule["primitives"], + "roots": rule["roots"], "source": rule["source"], "rationale": rule["rationale"], } @@ -428,9 +440,9 @@ def apply_waivers(findings: list[dict[str, Any]], configured: list[dict[str, Any accepted_primitives = {row["primitive"] for row in accepted} public_classification = finding.get("public_classification") public_primitives = ( - {public_classification["primitive"]} + set(public_classification["primitives"]) if isinstance(public_classification, dict) - and public_classification.get("primitive") in finding.get("primitive_ids", []) + and isinstance(public_classification.get("primitives"), list) else set() ) resolved_primitives = accepted_primitives | public_primitives diff --git a/scripts/ct/binsec.py b/scripts/ct/binsec.py index f554c12e..3653f06c 100755 --- a/scripts/ct/binsec.py +++ b/scripts/ct/binsec.py @@ -380,6 +380,7 @@ def write_report( reason: str, artifacts: dict[str, str], binsec_version: str | None, + binsec_sha256: str | None, timeout_seconds: int | None, smt_timeout_seconds: int | None, smt_solver: str | None, @@ -410,6 +411,7 @@ def write_report( "load_sections": load_sections or [], "sse_depth": kernel.get("sse_depth"), "binsec_version": binsec_version, + "binsec_sha256": binsec_sha256, "artifacts": artifacts, } path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") @@ -454,9 +456,11 @@ def main() -> int: binsec = find_binsec() binsec_version = None + binsec_sha256 = None if binsec is not None: version = run([binsec, "-version"]) binsec_version = (version.stdout or version.stderr).strip() or None + binsec_sha256 = sha256_file(Path(binsec).resolve()) if binsec is None and not args.allow_missing_binsec: print( @@ -490,6 +494,7 @@ def main() -> int: reason="binsec not found", artifacts=artifacts, binsec_version=binsec_version, + binsec_sha256=binsec_sha256, timeout_seconds=None, smt_timeout_seconds=None, smt_solver=args.smt_solver, @@ -519,6 +524,7 @@ def main() -> int: reason=f"missing symbol {kernel['symbol']}", artifacts=artifacts, binsec_version=binsec_version, + binsec_sha256=binsec_sha256, timeout_seconds=kernel_timeout, smt_timeout_seconds=kernel_smt_timeout, smt_solver=args.smt_solver, @@ -538,6 +544,7 @@ def main() -> int: reason="missing symbol ct_binsec_done", artifacts=artifacts, binsec_version=binsec_version, + binsec_sha256=binsec_sha256, timeout_seconds=kernel_timeout, smt_timeout_seconds=kernel_smt_timeout, smt_solver=args.smt_solver, @@ -557,6 +564,7 @@ def main() -> int: reason="BINSEC harness is PIE; static ELF proof binary required", artifacts=artifacts, binsec_version=binsec_version, + binsec_sha256=binsec_sha256, timeout_seconds=kernel_timeout, smt_timeout_seconds=kernel_smt_timeout, smt_solver=args.smt_solver, @@ -618,6 +626,7 @@ def main() -> int: reason=reason, artifacts=artifacts, binsec_version=binsec_version, + binsec_sha256=binsec_sha256, timeout_seconds=kernel_timeout, smt_timeout_seconds=kernel_smt_timeout, smt_solver=args.smt_solver, diff --git a/scripts/ct/dudect.sh b/scripts/ct/dudect.sh index 56cab757..4c9a7a8a 100755 --- a/scripts/ct/dudect.sh +++ b/scripts/ct/dudect.sh @@ -162,7 +162,7 @@ for tool in "$LLVM_OBJDUMP" "$LLVM_NM"; do exit 1 fi done -"$LLVM_OBJDUMP" --disassemble --reloc --demangle "$BINARY_PATH" > "$BINARY_DISASM_PATH" +"$LLVM_OBJDUMP" --disassemble --reloc --dynamic-reloc --demangle "$BINARY_PATH" > "$BINARY_DISASM_PATH" "$LLVM_NM" --defined-only --demangle "$BINARY_PATH" > "$BINARY_SYMBOLS_PATH" RUNNER_ARGS=(--out "$CSV_PATH") diff --git a/scripts/ct/dudect_report.py b/scripts/ct/dudect_report.py index c912b9e6..58a409de 100755 --- a/scripts/ct/dudect_report.py +++ b/scripts/ct/dudect_report.py @@ -335,6 +335,11 @@ "left_class": "fixed message", "right_class": "random same-length message", }, + "rsa_pkcs1v15_full_width_vs_short_canonical_crt_exponent": { + "primitive": "rsa.private_ops", + "left_class": "valid RSA-2048 key with 128-byte canonical CRT exponents", + "right_class": "valid same-factor-width RSA-2048 key with a 127-byte canonical dQ", + }, "rsa_oaep_decrypt_fixed_vs_random_plaintext": { "primitive": "rsa.private_ops", "left_class": "valid OAEP ciphertext for fixed plaintext", @@ -554,6 +559,75 @@ def raw_csv_rows(path: Path) -> dict[str, dict]: return rows +def owner_symbol_evidence( + symbols_text: str, + expected_symbols: set[str], +) -> tuple[dict[str, int], dict[int, str]]: + counts = {symbol: 0 for symbol in sorted(expected_symbols)} + addresses: dict[int, str] = {} + definition = re.compile(r"^\s*([0-9a-fA-F]+)\s+\S\s+_?(ct_entry_owner_eq_[0-9]+)\s*$") + for line in symbols_text.splitlines(): + if match := definition.match(line): + symbol = match.group(2) + if symbol in expected_symbols: + counts[symbol] += 1 + addresses[int(match.group(1), 16)] = symbol + return counts, addresses + + +def owner_call_site_counts( + disassembly_text: str, + expected_symbols: set[str], + symbols_by_address: dict[int, str], +) -> dict[str, int]: + counts = {symbol: 0 for symbol in sorted(expected_symbols)} + relative_relocations: dict[int, int] = {} + relative_relocation = re.compile( + r"^\s*([0-9a-fA-F]+)\s+R_[A-Z0-9_]+_RELATIVE\s+\*ABS\*\+0x([0-9a-fA-F]+)\s*$" + ) + for line in disassembly_text.splitlines(): + if match := relative_relocation.match(line): + relative_relocations[int(match.group(1), 16)] = int(match.group(2), 16) + + current_symbol = "" + function_label = re.compile(r"^[0-9a-fA-F]+ <(.+)>:$") + instruction_pattern = re.compile(r"\b(?:bl|brasl|call|callq|jal)\b") + for line in disassembly_text.splitlines(): + if match := function_label.match(line.strip()): + current_symbol = match.group(1).removeprefix("_") + continue + instruction = instruction_pattern.search(line) + if instruction is None: + continue + + called: set[str] = set() + for symbol in expected_symbols: + if re.search(rf"<_?{re.escape(symbol)}(?:\+[^>]*)?>", line): + called.add(symbol) + if target := re.search(r"\b0x([0-9a-fA-F]+)\b", line[instruction.end() :]): + if symbol := symbols_by_address.get(int(target.group(1), 16)): + called.add(symbol) + if slot := re.search(r"\*[^#]*#\s*0x([0-9a-fA-F]+)\b", line[instruction.end() :]): + if target_address := relative_relocations.get(int(slot.group(1), 16)): + if symbol := symbols_by_address.get(target_address): + called.add(symbol) + for symbol in called: + if current_symbol != symbol: + counts[symbol] += 1 + return counts + + +def linker_driver(command: str) -> str: + assignment = re.compile(r"[A-Za-z_][A-Za-z0-9_]*=.*") + for token in shlex.split(command): + if assignment.fullmatch(token): + continue + if token.startswith("-"): + break + return token + raise ValueError("DudeCT linker command does not identify the linker driver") + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--stdout", required=True, type=Path) @@ -593,27 +667,18 @@ def main() -> int: if not path.is_file(): raise ValueError(f"DudeCT evidence artifact missing: {path}") - symbol_counts = {symbol: 0 for symbol in expected_owner_symbols} - for line in args.binary_symbols.read_text().splitlines(): - if match := re.search(r"\b_?(ct_entry_owner_eq_[0-9]+)\b", line): - if match.group(1) in symbol_counts: - symbol_counts[match.group(1)] += 1 + symbol_counts, owner_symbols_by_address = owner_symbol_evidence( + args.binary_symbols.read_text(), + expected_owner_symbols, + ) wrong_symbol_counts = {symbol: count for symbol, count in symbol_counts.items() if count != 1} if wrong_symbol_counts: raise ValueError(f"DudeCT binary owner equality symbols must occur exactly once: {wrong_symbol_counts}") - owner_call_sites = {symbol: 0 for symbol in expected_owner_symbols} - current_symbol = "" - function_label = re.compile(r"^[0-9a-fA-F]+ <(.+)>:$") - for line in args.binary_disassembly.read_text(errors="replace").splitlines(): - if match := function_label.match(line.strip()): - current_symbol = match.group(1).removeprefix("_") - continue - instruction = re.search(r"\b(?:bl|brasl|call|callq|jal)\b", line) - if instruction is None: - continue - for symbol in expected_owner_symbols: - if current_symbol != symbol and re.search(rf"<_?{re.escape(symbol)}(?:\+[^>]*)?>", line): - owner_call_sites[symbol] += 1 + owner_call_sites = owner_call_site_counts( + args.binary_disassembly.read_text(errors="replace"), + expected_owner_symbols, + owner_symbols_by_address, + ) missing_call_sites = {symbol: count for symbol, count in owner_call_sites.items() if count < 1} if missing_call_sites: raise ValueError(f"DudeCT binary does not call every owner equality symbol: {missing_call_sites}") @@ -622,11 +687,7 @@ def main() -> int: (line for line in args.linker_command_log.read_text().splitlines() if '"-o"' in line), "", ) - linker_tokens = shlex.split(linker_command) - first_object = next((index for index, token in enumerate(linker_tokens) if token.endswith((".o", ".obj"))), None) - if first_object is None or first_object == 0: - raise ValueError("DudeCT linker command does not identify the linker driver") - linker = linker_tokens[first_object - 1] + linker = linker_driver(linker_command) linker_path_text = shutil.which(linker) if linker_path_text is None: raise ValueError(f"DudeCT linker driver is not resolvable: {linker}") diff --git a/scripts/ct/dudect_report_test.py b/scripts/ct/dudect_report_test.py new file mode 100644 index 00000000..53633614 --- /dev/null +++ b/scripts/ct/dudect_report_test.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Regression tests for architecture-specific DudeCT evidence parsing.""" + +from __future__ import annotations + +from dudect_report import linker_driver, owner_call_site_counts, owner_symbol_evidence + + +def expect_failure(action) -> None: + try: + action() + except ValueError: + return + raise AssertionError("malformed DudeCT evidence unexpectedly passed") + + +def main() -> None: + power_linker_command = ( + 'LC_ALL="C" PATH="/usr/local/bin:/usr/bin" VSLANG="1033" "cc" "-m64" ' + '"/tmp/rustc/first.o" "input.o" "-o" "/tmp/rscrypto-ct-dudect"' + ) + assert linker_driver(power_linker_command) == "cc" + assert linker_driver('LC_ALL="C" "/usr/bin/clang" "input.o" "-o" "output"') == "/usr/bin/clang" + expect_failure(lambda: linker_driver("")) + expect_failure(lambda: linker_driver('LC_ALL="C" "-m64" "input.o"')) + + expected = { + "ct_entry_owner_eq_16", + "ct_entry_owner_eq_32", + "ct_entry_owner_eq_48", + "ct_entry_owner_eq_64", + } + symbols = """\ +00000000001dad60 T ct_entry_owner_eq_16 +00000000001dadf0 T ct_entry_owner_eq_32 +00000000001daed0 T ct_entry_owner_eq_48 +00000000001daf60 T ct_entry_owner_eq_64 +""" + symbol_counts, symbols_by_address = owner_symbol_evidence(symbols, expected) + assert symbol_counts == {symbol: 1 for symbol in sorted(expected)} + assert symbols_by_address[0x1DAD60] == "ct_entry_owner_eq_16" + malformed_counts, malformed_addresses = owner_symbol_evidence( + "ct_entry_owner_eq_16\n", + expected, + ) + assert malformed_counts == {symbol: 0 for symbol in sorted(expected)} + assert malformed_addresses == {} + + s390x_disassembly = """\ +00000000000382c0 : + 382f0: c0 e5 00 0d 15 38 brasl %r14, 0x1dad60 + 3873a: c0 e5 00 0d 13 5b brasl %r14, 0x1dadf0 + 38b7a: c0 e5 00 0d 11 ab brasl %r14, 0x1daed0 + 38fba: c0 e5 00 0d 0f d3 brasl %r14, 0x1daf60 +""" + assert owner_call_site_counts(s390x_disassembly, expected, symbols_by_address) == { + symbol: 1 for symbol in sorted(expected) + } + + unresolved = s390x_disassembly.replace("0x1dad60", "0x1dad61") + unresolved_counts = owner_call_site_counts(unresolved, expected, symbols_by_address) + assert unresolved_counts["ct_entry_owner_eq_16"] == 0 + + symbolic_disassembly = """\ +0000000000001000 : + 1000: e8 0b 00 00 00 callq 0x1010 +""" + symbolic_counts = owner_call_site_counts(symbolic_disassembly, expected, {}) + assert symbolic_counts["ct_entry_owner_eq_16"] == 1 + + x86_got_disassembly = """\ +DYNAMIC RELOCATION RECORDS +00000000003733a0 R_X86_64_RELATIVE *ABS*+0x1dad60 +00000000003733a8 R_X86_64_RELATIVE *ABS*+0x1dadf0 +00000000003733b0 R_X86_64_RELATIVE *ABS*+0x1daed0 +00000000003733b8 R_X86_64_RELATIVE *ABS*+0x1daf60 + +0000000000010000 : + 10010: ff 15 8a 33 36 00 callq *0x36338a(%rip) # 0x3733a0 + 10016: ff 15 8c 33 36 00 callq *0x36338c(%rip) # 0x3733a8 + 1001c: ff 15 8e 33 36 00 callq *0x36338e(%rip) # 0x3733b0 + 10022: ff 15 90 33 36 00 callq *0x363390(%rip) # 0x3733b8 +""" + assert owner_call_site_counts(x86_got_disassembly, expected, symbols_by_address) == { + symbol: 1 for symbol in sorted(expected) + } + + missing_relocation = x86_got_disassembly.replace( + "00000000003733a0 R_X86_64_RELATIVE *ABS*+0x1dad60\n", + "", + ) + missing_relocation_counts = owner_call_site_counts(missing_relocation, expected, symbols_by_address) + assert missing_relocation_counts["ct_entry_owner_eq_16"] == 0 + + +if __name__ == "__main__": + main() diff --git a/scripts/ct/evidence_validation_test.py b/scripts/ct/evidence_validation_test.py index f0c32d73..d2e6d1cf 100644 --- a/scripts/ct/evidence_validation_test.py +++ b/scripts/ct/evidence_validation_test.py @@ -144,6 +144,14 @@ def required_target_without_snapshot(manifest) -> None: in manifest_errors(required_target_without_snapshot) ) + def public_operand_primitive_without_root(manifest) -> None: + manifest["asm_public_operand"][0]["primitives"].append("owner_equality.fixed") + + assert ( + "asm_public_operand[0] has no root owned by primitive owner_equality.fixed" + in manifest_errors(public_operand_primitive_without_root) + ) + with tempfile.TemporaryDirectory() as temporary: temporary_path = Path(temporary) hashes = temporary_path / "hashes.txt" @@ -227,8 +235,8 @@ def required_target_without_snapshot(manifest) -> None: "waived": False, } rule = { - "primitive": "password.argon2i", - "root": "ct_entry_argon2i_verify", + "primitives": ["password.argon2i"], + "roots": ["ct_entry_argon2i_verify"], "symbol": finding["symbol"], "kind": finding["kind"], "max_count": 1, @@ -242,20 +250,20 @@ def required_target_without_snapshot(manifest) -> None: functions = { finding["symbol"]: FunctionBody(finding["symbol"], Path("fixture"), 0, []), - rule["root"]: FunctionBody(rule["root"], Path("fixture"), 0, []), + rule["roots"][0]: FunctionBody(rule["roots"][0], Path("fixture"), 0, []), } symbol_summary = summarize(set(functions), functions, [finding])[finding["symbol"]] assert symbol_summary["unwaived_fail_count"] == 0 assert symbol_summary["accepted_count"] == 1 - closures = {rule["primitive"]: {rule["root"]: set(functions)}} - primitive_summary = summarize_closure(closures, functions, [finding])[rule["primitive"]] + closures = {rule["primitives"][0]: {rule["roots"][0]: set(functions)}} + primitive_summary = summarize_closure(closures, functions, [finding])[rule["primitives"][0]] assert primitive_summary["unwaived_fail_count"] == 0 assert primitive_summary["accepted_count"] == 1 mixed = dict( finding, - primitive_ids=[rule["primitive"], "fixture.unresolved"], + primitive_ids=[rule["primitives"][0], "fixture.unresolved"], operand_class="unproven", disposition="needs-fix", waived=False, @@ -266,12 +274,12 @@ def required_target_without_snapshot(manifest) -> None: assert mixed["operand_class"] == "unproven" and mixed["disposition"] == "needs-fix" mixed_closures = { - rule["primitive"]: {rule["root"]: set(functions)}, - "fixture.unresolved": {rule["root"]: set(functions)}, + rule["primitives"][0]: {rule["roots"][0]: set(functions)}, + "fixture.unresolved": {rule["roots"][0]: set(functions)}, } mixed_summary = summarize_closure(mixed_closures, functions, [mixed]) - assert mixed_summary[rule["primitive"]]["unwaived_fail_count"] == 0 - assert mixed_summary[rule["primitive"]]["accepted_count"] == 1 + assert mixed_summary[rule["primitives"][0]]["unwaived_fail_count"] == 0 + assert mixed_summary[rule["primitives"][0]]["accepted_count"] == 1 assert mixed_summary["fixture.unresolved"]["unwaived_fail_count"] == 1 assert mixed_summary["fixture.unresolved"]["accepted_count"] == 0 diff --git a/scripts/ct/full.py b/scripts/ct/full.py index 4484b02f..2aee887f 100755 --- a/scripts/ct/full.py +++ b/scripts/ct/full.py @@ -1249,6 +1249,7 @@ def main() -> int: "harness_elf_type": report.get("harness_elf_type"), "load_sections": report.get("load_sections"), "binsec_version": report.get("binsec_version"), + "binsec_sha256": report.get("binsec_sha256"), "artifacts": report.get("artifacts", {}), "artifact_dir": str(report_path.parent.relative_to(out_dir)), } diff --git a/scripts/ct/validate.py b/scripts/ct/validate.py index b644a5f0..1ca854a8 100755 --- a/scripts/ct/validate.py +++ b/scripts/ct/validate.py @@ -516,17 +516,50 @@ def validate_manifest(root: Path, selected_target: str, errors: list[str], warni fail(errors, f"harness {harness.get('name', '')} covers unknown primitive {covered}") for index, rule in enumerate(ct.get("asm_public_operand", [])): - required_fields = {"primitive", "root", "symbol", "kind", "max_count", "source", "rationale"} + required_fields = {"primitives", "roots", "symbol", "kind", "max_count", "source", "rationale"} if set(rule) != required_fields: fail(errors, f"asm_public_operand[{index}] has incomplete or unknown fields") continue - primitive_id = rule.get("primitive") - if primitive_id not in primitive_ids: - fail(errors, f"asm_public_operand[{index}] references unknown primitive {primitive_id!r}") + rule_primitives = rule.get("primitives") + rule_roots = rule.get("roots") + if ( + not isinstance(rule_primitives, list) + or not rule_primitives + or any(not isinstance(value, str) or not value for value in rule_primitives) + or len(rule_primitives) != len(set(rule_primitives)) + ): + fail(errors, f"asm_public_operand[{index}] primitives must be a non-empty list of unique strings") + continue + if ( + not isinstance(rule_roots, list) + or not rule_roots + or any(not isinstance(value, str) or not value for value in rule_roots) + or len(rule_roots) != len(set(rule_roots)) + ): + fail(errors, f"asm_public_operand[{index}] roots must be a non-empty list of unique strings") continue - primitive = next(row for row in ct.get("primitive", []) if row.get("id") == primitive_id) - if rule.get("root") not in primitive.get("harness", {}).get("symbols", []): - fail(errors, f"asm_public_operand[{index}] root is not owned by primitive {primitive_id}") + unknown_primitives = sorted(set(rule_primitives) - primitive_ids) + if unknown_primitives: + fail(errors, f"asm_public_operand[{index}] references unknown primitives: {', '.join(unknown_primitives)}") + continue + owned_roots = { + symbol + for primitive in ct.get("primitive", []) + if primitive.get("id") in rule_primitives + for symbol in primitive.get("harness", {}).get("symbols", []) + } + unknown_roots = sorted(set(rule_roots) - owned_roots) + if unknown_roots: + fail(errors, f"asm_public_operand[{index}] roots are not owned by its primitives: {', '.join(unknown_roots)}") + for primitive_id in rule_primitives: + primitive_roots = { + symbol + for primitive in ct.get("primitive", []) + if primitive.get("id") == primitive_id + for symbol in primitive.get("harness", {}).get("symbols", []) + } + if set(rule_roots).isdisjoint(primitive_roots): + fail(errors, f"asm_public_operand[{index}] has no root owned by primitive {primitive_id}") if rule.get("kind") not in {"variable_latency_division", "variable_latency_multiply"}: fail(errors, f"asm_public_operand[{index}] kind is not a public-operand-classifiable instruction") max_count = rule.get("max_count") diff --git a/scripts/ct/validate_release_evidence.py b/scripts/ct/validate_release_evidence.py index e9ebe9ac..5d7d4546 100755 --- a/scripts/ct/validate_release_evidence.py +++ b/scripts/ct/validate_release_evidence.py @@ -343,6 +343,7 @@ def validate_binsec( "harness_manifest_sha256": sha256_git_file(root, commit, harness_manifest_path), "harness_lockfile_sha256": sha256_git_file(root, commit, harness_lockfile_path), } + binsec_hashes: set[str] = set() for row in required_kernel_rows: for key, expected in { "backend": "llvm", @@ -366,6 +367,11 @@ def validate_binsec( fail(f"{suffix} BINSEC {row.get('kernel')} does not use a static executable proof driver") if not isinstance(row.get("binsec_version"), str) or not row.get("binsec_version"): fail(f"{suffix} BINSEC {row.get('kernel')} lacks the BINSEC version") + binsec_sha256 = row.get("binsec_sha256") + if not isinstance(binsec_sha256, str) or re.fullmatch(r"[0-9a-f]{64}", binsec_sha256) is None: + fail(f"{suffix} BINSEC {row.get('kernel')} lacks the BINSEC executable hash") + else: + binsec_hashes.add(binsec_sha256) if ".text" not in row.get("load_sections", []): fail(f"{suffix} BINSEC {row.get('kernel')} does not load the proof driver's text section") component_artifacts = row.get("artifacts", {}) @@ -375,6 +381,8 @@ def validate_binsec( record = report_artifacts.get(path) if not isinstance(digest, str) or record is None or record.get("sha256") != digest: fail(f"{suffix} BINSEC {row.get('kernel')} does not bind {required_name}") + if len(binsec_hashes) != 1: + fail(f"{suffix} required BINSEC kernels do not bind one exact analyzer executable") else: require_equal(binsec.get("enabled"), False, f"{suffix} BINSEC enabled") require_equal(binsec.get("policy"), "unsupported", f"{suffix} BINSEC policy") diff --git a/scripts/lib/rail-plan.sh b/scripts/lib/rail-plan.sh index 4fbbef04..c9e9418d 100644 --- a/scripts/lib/rail-plan.sh +++ b/scripts/lib/rail-plan.sh @@ -160,6 +160,19 @@ rail_scope_json() { if [[ -n "${RAIL_SCOPE_JSON:-}" ]]; then scope_output="$RAIL_SCOPE_JSON" + if [[ -n "${RAIL_SURFACES_JSON:-}" ]]; then + if ! scope_output="$( + jq -ce --argjson surfaces "$RAIL_SURFACES_JSON" ' + if has("surfaces") and .surfaces != $surfaces then + empty + else + . + { surfaces: $surfaces } + end + ' <<<"$scope_output" 2>/dev/null + )"; then + return 1 + fi + fi elif [[ -n "${RAIL_SCOPE_JSON_CACHE:-}" ]]; then scope_output="$RAIL_SCOPE_JSON_CACHE" else diff --git a/scripts/test/test-rsa-macos-asm.sh b/scripts/test/test-rsa-macos-asm.sh new file mode 100755 index 00000000..2fbcc4d0 --- /dev/null +++ b/scripts/test/test-rsa-macos-asm.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$REPO_ROOT" + +die() { + echo "RSA macOS assembly evidence error: $*" >&2 + exit 1 +} + +assert_single_libtest() { + local test_name=$1 + shift + + local listing count + if ! listing=$("$@" --list); then + die "unable to list the test harness containing $test_name" + fi + count=$(printf '%s\n' "$listing" | awk -v expected="$test_name: test" '$0 == expected { count++ } END { print count + 0 }') + [[ "$count" -eq 1 ]] || die "expected exactly one libtest named $test_name; found $count" +} + +[[ $# -eq 0 ]] || die "usage: scripts/test/test-rsa-macos-asm.sh" + +evidence_dir="$REPO_ROOT/target/rsa-macos-asm" +mkdir -p "$evidence_dir" +{ + uname -a + [[ "$(uname -s)" == Darwin ]] || die "local RSA macOS assembly evidence requires macOS" + [[ "$(uname -m)" == arm64 ]] || die "local RSA macOS assembly evidence requires Arm64" + [[ "$(rustc -vV | sed -n 's/^host: //p')" == aarch64-apple-darwin ]] \ + || die "local RSA macOS assembly evidence requires the aarch64-apple-darwin Rust host" + cpu_brand=$(sysctl -n machdep.cpu.brand_string) \ + || die "unable to read the macOS CPU identity" + hypervisor_present=$(sysctl -n kern.hv_vmm_present) \ + || die "unable to determine whether macOS is virtualized" + printf 'CPU: %s\nHypervisor present: %s\n' "$cpu_brand" "$hypervisor_present" + [[ "$hypervisor_present" == 0 ]] \ + || die "local RSA macOS assembly evidence requires a physical Apple Silicon Mac" + rustc -vV + + assert_single_libtest \ + auth::rsa::tests::aarch64_macos_rsa_montgomery_asm_matches_portable_across_supported_widths \ + cargo test --locked --features rsa,diag,getrandom --lib -- + cargo test --locked --features rsa,diag,getrandom --lib \ + auth::rsa::tests::aarch64_macos_rsa_montgomery_asm_matches_portable_across_supported_widths \ + -- --exact --nocapture + assert_single_libtest \ + auth::rsa::tests::aarch64_macos_rsa_montgomery_asm_matches_portable_across_supported_widths \ + cargo test --locked --release --features rsa,diag,getrandom --lib -- + cargo test --locked --release --features rsa,diag,getrandom --lib \ + auth::rsa::tests::aarch64_macos_rsa_montgomery_asm_matches_portable_across_supported_widths \ + -- --exact --nocapture + + build_output=$(cargo test --locked --release --features rsa,diag \ + --test rsa_public_key --no-run --message-format=json) + binary=$(printf '%s\n' "$build_output" \ + | sed -n 's/.*"executable":"\([^"]*rsa_public_key-[^"]*\)".*/\1/p' \ + | tail -n 1) + [[ -n "$binary" && -x "$binary" ]] \ + || die "unable to resolve the optimized rsa_public_key test binary" + printf 'Optimized RSA test binary: %s\n' "$binary" + assert_single_libtest public_operation_montgomery_candidates_match_current_path "$binary" + "$binary" public_operation_montgomery_candidates_match_current_path --exact --nocapture + + binary_description=$(file "$binary") || die "unable to inspect the optimized rsa_public_key test binary" + [[ "$binary_description" == *"Mach-O 64-bit executable arm64"* ]] \ + || die "optimized rsa_public_key test binary is not Arm64 Mach-O" + binary_symbols=$(nm -m "$binary") || die "unable to read the optimized rsa_public_key symbol table" + [[ "$binary_symbols" == *"_rscrypto_rsa_bn_mul_mont_words_apple"* ]] \ + || die "optimized rsa_public_key test binary lacks the Apple Montgomery multiply" + [[ "$binary_symbols" == *"_rscrypto_rsa_mont_reduce_cios_32_aarch64_apple_darwin"* ]] \ + || die "optimized rsa_public_key test binary lacks the Apple 32-word Montgomery reduction" + [[ "$binary_symbols" == *"_rscrypto_rsa_mont_reduce_cios_words_aarch64_apple_darwin"* ]] \ + || die "optimized rsa_public_key test binary lacks the Apple generic Montgomery reduction" +} 2>&1 | tee "$evidence_dir/evidence.log" diff --git a/src/aead/aegis256.rs b/src/aead/aegis256.rs index 9bd3e574..849441d5 100644 --- a/src/aead/aegis256.rs +++ b/src/aead/aegis256.rs @@ -2,10 +2,8 @@ //! AEGIS-256 authenticated encryption (draft-irtf-cfrg-aegis-aead). //! -//! High-performance AES-based AEAD with a 256-bit key, 256-bit nonce, -//! and 128-bit authentication tag. Uses raw AES round functions (not full -//! AES encryption), achieving ~2-3x the throughput of AES-256-GCM on -//! hardware with AES-NI or AES-CE. +//! AES-round-based AEAD with a 256-bit key, 256-bit nonce, and 128-bit +//! authentication tag. use core::fmt; @@ -212,6 +210,7 @@ fn finalize(s: &mut State, ad_len: usize, msg_len: usize) -> [u8; TAG_SIZE] { tag = xor_block(&tag, &s[3]); tag = xor_block(&tag, &s[4]); tag = xor_block(&tag, &s[5]); + ct::zeroize(&mut t); tag } @@ -283,10 +282,8 @@ define_aead_tag_type!(Aegis256Tag, TAG_SIZE, "AEGIS-256 128-bit authentication t /// AEGIS-256 authenticated encryption with associated data. /// -/// High-performance AES-based AEAD with a 256-bit key, 256-bit nonce, -/// and 128-bit authentication tag. On hardware with AES round instructions -/// (AES-NI, AES-CE, POWER8 vcipher), AEGIS-256 achieves -/// ~2-3x the throughput of AES-256-GCM. +/// AES-round-based AEAD with a 256-bit key, 256-bit nonce, and 128-bit +/// authentication tag. /// /// # Security /// @@ -432,10 +429,14 @@ fn encrypt_portable(key: &[u8; KEY_SIZE], nonce: &[u8; NONCE_SIZE], aad: &[u8], buffer[offset..].copy_from_slice(&ct[..tail_len]); } - finalize(&mut s, aad.len(), msg_len) + let tag = finalize(&mut s, aad.len(), msg_len); + ct::zeroize(s.as_flattened_mut()); + tag } #[cfg(feature = "diag")] +#[unsafe(no_mangle)] +#[inline(never)] #[must_use] pub fn diag_aegis256_update_portable( key: &[u8; KEY_SIZE], @@ -444,7 +445,9 @@ pub fn diag_aegis256_update_portable( ) -> [u8; TAG_SIZE] { let mut s = init(key, nonce); update(&mut s, block); - finalize(&mut s, 0, BLOCK_SIZE) + let tag = finalize(&mut s, 0, BLOCK_SIZE); + ct::zeroize(s.as_flattened_mut()); + tag } fn decrypt_portable(key: &[u8; KEY_SIZE], nonce: &[u8; NONCE_SIZE], aad: &[u8], buffer: &mut [u8]) -> [u8; TAG_SIZE] { @@ -479,7 +482,9 @@ fn decrypt_portable(key: &[u8; KEY_SIZE], nonce: &[u8; NONCE_SIZE], aad: &[u8], buffer[offset..].copy_from_slice(&pt_pad[..tail_len]); } - finalize(&mut s, aad.len(), ct_len) + let tag = finalize(&mut s, aad.len(), ct_len); + ct::zeroize(s.as_flattened_mut()); + tag } impl Aead for Aegis256 { @@ -526,13 +531,9 @@ impl Aead for Aegis256 { let key = self.key.as_bytes(); let nonce = nonce.as_bytes(); - // NOTE: VAES-256 (`ni_wide`) is intentionally NOT dispatched here. - // AEGIS-256's update is a serial chain of 6 AES rounds reading old state. - // VAES-256 packs into 3 YMM registers but requires 3 cross-lane shuffles - // (`vperm2i128`, 3-cycle latency) before each set of 3 VAESENC, adding - // ~3 cycles to the critical path. On Zen4 with 2 AES ports: AES-NI steady- - // state is ~5 cyc/block; VAES-256 is ~8 cyc/block. AES-NI wins for serial - // update chains (unlike AES-GCM where blocks are independent). + // AEGIS-256 updates six dependent state lanes. The wide VAES kernel packs + // them into three YMM registers and needs cross-lane shuffles between + // updates; this path keeps the serial state in XMM registers. #[cfg(target_arch = "x86_64")] if self.backend == AeadBackend::X86Aesni { // SAFETY: backend resolution confirmed AES-NI + AVX are available. diff --git a/src/aead/aegis256/aarch64_ce.rs b/src/aead/aegis256/aarch64_ce.rs index ce5485f9..d8262b58 100644 --- a/src/aead/aegis256/aarch64_ce.rs +++ b/src/aead/aegis256/aarch64_ce.rs @@ -21,13 +21,9 @@ unsafe fn store(v: uint8x16_t, out: &mut [u8; BLOCK_SIZE]) { // ── Register-based helpers ────────────────────────────────────────────── // // ARM AESE applies AddRoundKey before SubBytes, so both operand forms below -// compute the same AESENC-compatible round. Apple cores benchmark faster with -// the state as AESE's destructive data operand; non-Apple targets use the -// libaegis operand form, preserving old state registers for the next pipeline -// assignments on Neoverse-class cores. -// -// Neoverse V1/V2 has 2 crypto pipelines; register-based code gives the -// OOO engine maximum scheduling freedom across both pipes. +// compute the same AESENC-compatible round. Apple targets use the state as +// AESE's destructive data operand; non-Apple targets use the libaegis operand +// form, preserving old state registers for the next pipeline assignments. #[target_feature(enable = "aes,neon")] #[inline] diff --git a/src/aead/aes.rs b/src/aead/aes.rs index aefb1623..e77afab3 100644 --- a/src/aead/aes.rs +++ b/src/aead/aes.rs @@ -500,7 +500,7 @@ pub(crate) fn aes256_expand_key(key: &[u8; KEY_SIZE]) -> Aes256EncKey { if crate::platform::caps().has(crate::platform::caps::aarch64::AES) { return Aes256EncKey { // SAFETY: AES-CE availability verified via HWCAP above. - // Uses AESE for SubWord — ~750x faster than the algebraic GF(2^8) S-box. + // Uses AESE rather than the portable algebraic S-box for SubWord. inner: KeyInner::Aarch64Aes(unsafe { ce::expand_key(key) }), }; } @@ -4182,8 +4182,8 @@ pub(crate) unsafe fn aes128_ctr32_encrypt_be_aarch64_ghash( ]); let mut offset = 0usize; - // Short messages are faster if GHASH consumes ciphertext registers immediately. The lagged - // pipeline below wins once there is enough work to hide GHASH behind the next AES chunk. + // The configured short-input path feeds ciphertext registers directly to + // GHASH; longer inputs use the lagged AES/GHASH pipeline. if data.len() <= 256 { while offset.strict_add(128) <= data.len() { let end = offset.strict_add(128); diff --git a/src/aead/aes/aarch64_ce.rs b/src/aead/aes/aarch64_ce.rs index f0a128ef..b22c1cd5 100644 --- a/src/aead/aes/aarch64_ce.rs +++ b/src/aead/aes/aarch64_ce.rs @@ -31,8 +31,8 @@ impl Drop for CeRoundKeys { /// The AESE instruction applies SubBytes to all 16 bytes. By broadcasting /// a 32-bit word to all 4 columns of the AES state, ShiftRows becomes a /// no-op (all columns identical), so `AESE(broadcast(w), 0)` = `SubWord(w)`. -/// This replaces ~1560 GF(2^8) field operations per SubWord call with a -/// single AESE instruction (~1 cycle on Neoverse V1/V2). +/// This replaces the portable algebraic S-box evaluation with one AESE +/// instruction per SubWord operation. #[target_feature(enable = "aes,neon")] #[inline] /// # Safety diff --git a/src/aead/aes128gcm.rs b/src/aead/aes128gcm.rs index ae947731..f3e860fb 100644 --- a/src/aead/aes128gcm.rs +++ b/src/aead/aes128gcm.rs @@ -213,6 +213,14 @@ impl Aes128Gcm { // GCM construction internals (NIST SP 800-38D) +struct GhashAccumulator(u128); + +impl Drop for GhashAccumulator { + fn drop(&mut self) { + ct::zeroize_words(core::slice::from_mut(&mut self.0)); + } +} + /// Build the initial counter block J0 and first CTR block for a 96-bit IV. /// /// J0 = IV || 0x00000001 (NIST SP 800-38D § 7.1, when len(IV) = 96). @@ -237,15 +245,15 @@ fn compute_tag( aad: &[u8], ciphertext: &[u8], ) -> Result<[u8; TAG_SIZE], LengthOverflow> { - let mut acc = 0u128; - acc = ghash_update_padded(acc, h_polyval, aad); - acc = ghash_update_padded(acc, h_polyval, ciphertext); + let mut acc = GhashAccumulator(0); + acc.0 = ghash_update_padded(acc.0, h_polyval, aad); + acc.0 = ghash_update_padded(acc.0, h_polyval, ciphertext); let length_block = super::AeadByteLengths::try_new_bit_lengths(aad.len(), ciphertext.len())?.to_be_bits_block(); - acc ^= u128::from_be_bytes(length_block); - acc = polyval::clmul128_reduce(acc, h_polyval); + acc.0 ^= u128::from_be_bytes(length_block); + acc.0 = polyval::clmul128_reduce(acc.0, h_polyval); - Ok(encrypt_j0_tag(ek, j0, acc)) + Ok(encrypt_j0_tag(ek, j0, acc.0)) } #[inline] @@ -358,8 +366,8 @@ fn compute_tag_short_wide( let mut blocks = [0u128; 4]; let start = 4usize.strict_sub(block_count); blocks[start..].copy_from_slice(&sequence[..block_count]); - let acc = polyval::accumulate_4blocks(0, h_polyval, h_powers_rev, &blocks); - Ok(Some(encrypt_j0_tag(ek, j0, acc))) + let acc = GhashAccumulator(polyval::accumulate_4blocks(0, h_polyval, h_powers_rev, &blocks)); + Ok(Some(encrypt_j0_tag(ek, j0, acc.0))) } #[inline(always)] @@ -381,24 +389,26 @@ pub fn diag_aes128gcm_ctr32_be(cipher: &Aes128Gcm, nonce: &Nonce96, plaintext: & } #[cfg(feature = "diag")] +#[unsafe(no_mangle)] +#[inline(never)] #[must_use] pub fn diag_aes128gcm_ghash(cipher: &Aes128Gcm, aad: &[u8], ciphertext: &[u8]) -> [u8; 16] { let h_polyval = cipher.h_powers_rev[3]; - let mut acc = 0u128; + let mut acc = GhashAccumulator(0); if should_use_wide_ghash(cipher.backend, aad.len(), ciphertext.len()) { - acc = ghash_update_padded_wide(acc, h_polyval, &cipher.h_powers_rev, aad); - acc = ghash_update_padded_wide(acc, h_polyval, &cipher.h_powers_rev, ciphertext); + acc.0 = ghash_update_padded_wide(acc.0, h_polyval, &cipher.h_powers_rev, aad); + acc.0 = ghash_update_padded_wide(acc.0, h_polyval, &cipher.h_powers_rev, ciphertext); } else { - acc = ghash_update_padded(acc, h_polyval, aad); - acc = ghash_update_padded(acc, h_polyval, ciphertext); + acc.0 = ghash_update_padded(acc.0, h_polyval, aad); + acc.0 = ghash_update_padded(acc.0, h_polyval, ciphertext); } let length_block = match super::AeadByteLengths::try_new_bit_lengths(aad.len(), ciphertext.len()) { Ok(lengths) => lengths.to_be_bits_block(), Err(_) => return [0u8; 16], }; - acc ^= u128::from_be_bytes(length_block); - acc = polyval::clmul128_reduce(acc, h_polyval); - acc.to_be_bytes() + acc.0 ^= u128::from_be_bytes(length_block); + acc.0 = polyval::clmul128_reduce(acc.0, h_polyval); + acc.0.to_be_bytes() } #[cfg(feature = "diag")] @@ -433,16 +443,16 @@ fn compute_tag_wide( aad: &[u8], ciphertext: &[u8], ) -> Result<[u8; TAG_SIZE], LengthOverflow> { - let mut acc = 0u128; - acc = ghash_update_padded_wide(acc, h_polyval, h_powers_rev, aad); - acc = ghash_update_padded_wide(acc, h_polyval, h_powers_rev, ciphertext); + let mut acc = GhashAccumulator(0); + acc.0 = ghash_update_padded_wide(acc.0, h_polyval, h_powers_rev, aad); + acc.0 = ghash_update_padded_wide(acc.0, h_polyval, h_powers_rev, ciphertext); // Length block. let length_block = super::AeadByteLengths::try_new_bit_lengths(aad.len(), ciphertext.len())?.to_be_bits_block(); - acc ^= u128::from_be_bytes(length_block); - acc = polyval::clmul128_reduce(acc, h_polyval); + acc.0 ^= u128::from_be_bytes(length_block); + acc.0 = polyval::clmul128_reduce(acc.0, h_polyval); - Ok(encrypt_j0_tag(ek, j0, acc)) + Ok(encrypt_j0_tag(ek, j0, acc.0)) } /// Update GHASH using VPCLMUL without scalar block packing. @@ -710,18 +720,18 @@ impl Aead for Aes128Gcm { // SAFETY: x86 GHASH AAD path because: // 1. Backend resolution selected X86VaesVpclmul only after CPUID confirmed VPCLMULQDQ. // 2. `aad` is a valid byte slice; padding is handled inside the helper. - let mut acc = unsafe { ghash_update_padded_wide_x86(0, h_polyval, tables.h_powers_rev, aad) }; + let mut acc = GhashAccumulator(unsafe { ghash_update_padded_wide_x86(0, h_polyval, tables.h_powers_rev, aad) }); // SAFETY: fused x86 AES-GCM sealing because: // 1. Backend resolution selected X86VaesVpclmul only after CPUID confirmed VAES, VPCLMULQDQ, and // AES-NI. // 2. The helper encrypts `buffer` in place and folds the resulting ciphertext into GHASH. - acc = unsafe { aes::aes128_ctr32_encrypt_be_wide_ghash(&self.ek, &ctr_block, buffer, acc, tables) }; - acc ^= u128::from_be_bytes(length_block); + acc.0 = unsafe { aes::aes128_ctr32_encrypt_be_wide_ghash(&self.ek, &ctr_block, buffer, acc.0, tables) }; + acc.0 ^= u128::from_be_bytes(length_block); // SAFETY: x86 GHASH final multiply because: // 1. Backend resolution selected X86VaesVpclmul only after CPUID confirmed PCLMULQDQ. // 2. `acc` and `h_polyval` are initialized GHASH field elements. - acc = unsafe { polyval::x86_clmul128_reduce_inline(acc, h_polyval) }; - let tag_bytes = encrypt_j0_tag(&self.ek, &j0, acc); + acc.0 = unsafe { polyval::x86_clmul128_reduce_inline(acc.0, h_polyval) }; + let tag_bytes = encrypt_j0_tag(&self.ek, &j0, acc.0); return Ok(Aes128GcmTag::from_bytes(tag_bytes)); } @@ -731,25 +741,25 @@ impl Aead for Aes128Gcm { && crate::platform::caps().has(crate::platform::caps::x86::PCLMUL_READY) { let h_polyval = self.h_powers_rev[3]; - let mut acc = ghash_update_padded_wide(0, h_polyval, &self.h_powers_rev, aad); + let mut acc = GhashAccumulator(ghash_update_padded_wide(0, h_polyval, &self.h_powers_rev, aad)); // SAFETY: fused x86 AES-NI/PCLMUL sealing because: // 1. Backend resolution selected X86AesniPclmul only after CPUID confirmed AES-NI and PCLMULQDQ. // 2. The extra PCLMUL_READY check confirms SSSE3 for in-register GHASH byte reversal. // 3. The helper encrypts `buffer` in place and folds ciphertext into GHASH. - acc = unsafe { + acc.0 = unsafe { aes::aes128_ctr32_encrypt_be_aesni_pclmul_ghash( &self.ek, &ctr_block, buffer, - acc, + acc.0, h_polyval, &self.h_powers_rev, ) }; - acc ^= u128::from_be_bytes(length_block); + acc.0 ^= u128::from_be_bytes(length_block); // SAFETY: x86 GHASH final multiply because PCLMUL_READY was checked above. - acc = unsafe { polyval::x86_clmul128_reduce_inline(acc, h_polyval) }; - let tag_bytes = encrypt_j0_tag(&self.ek, &j0, acc); + acc.0 = unsafe { polyval::x86_clmul128_reduce_inline(acc.0, h_polyval) }; + let tag_bytes = encrypt_j0_tag(&self.ek, &j0, acc.0); return Ok(Aes128GcmTag::from_bytes(tag_bytes)); } @@ -763,7 +773,8 @@ impl Aead for Aes128Gcm { // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed AES-CE // and PMULL. // 2. `aad` is a valid byte slice; padding is handled inside the helper. - let mut acc = unsafe { ghash_update_padded_wide_aarch64(0, h_polyval, &self.h_powers_rev, aad) }; + let mut acc = + GhashAccumulator(unsafe { ghash_update_padded_wide_aarch64(0, h_polyval, &self.h_powers_rev, aad) }); // SAFETY: fused intrinsic AArch64 AES-GCM sealing because: // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed AES-CE // and PMULL. @@ -781,13 +792,13 @@ impl Aead for Aes128Gcm { }; // SAFETY: backend selection confirmed AES-CE/PMULL, and the helper encrypts the valid buffer // in place while folding ciphertext into GHASH. - acc = unsafe { aes::aes128_ctr32_encrypt_be_aarch64_ghash(&self.ek, &ctr_block, buffer, acc, &tables) }; - acc ^= u128::from_be_bytes(length_block); + acc.0 = unsafe { aes::aes128_ctr32_encrypt_be_aarch64_ghash(&self.ek, &ctr_block, buffer, acc.0, &tables) }; + acc.0 ^= u128::from_be_bytes(length_block); // SAFETY: aarch64 GHASH final multiply because: // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed PMULL. // 2. `acc` and `h_polyval` are initialized GHASH field elements. - acc = unsafe { polyval::aarch64_clmul128_reduce_inline(acc, h_polyval) }; - let tag_bytes = encrypt_j0_tag(&self.ek, &j0, acc); + acc.0 = unsafe { polyval::aarch64_clmul128_reduce_inline(acc.0, h_polyval) }; + let tag_bytes = encrypt_j0_tag(&self.ek, &j0, acc.0); return Ok(Aes128GcmTag::from_bytes(tag_bytes)); } @@ -798,18 +809,18 @@ impl Aead for Aes128Gcm { // 1. Backend resolution selected `Power8Crypto` only after runtime detection confirmed POWER8 // crypto. // 2. `aad` is a valid byte slice; padding is handled inside the helper. - let mut acc = unsafe { ghash_update_padded_wide_ppc(0, h_polyval, &self.h_powers_rev, aad) }; + let mut acc = GhashAccumulator(unsafe { ghash_update_padded_wide_ppc(0, h_polyval, &self.h_powers_rev, aad) }); // SAFETY: fused POWER AES-GCM sealing because: // 1. Backend resolution selected `Power8Crypto` only after runtime detection confirmed POWER8 // crypto. // 2. The helper encrypts `buffer` in place and folds ciphertext into GHASH. - acc = unsafe { - aes::aes128_ctr32_encrypt_be_ppc_ghash(&self.ek, &ctr_block, buffer, acc, h_polyval, &self.h_powers_rev) + acc.0 = unsafe { + aes::aes128_ctr32_encrypt_be_ppc_ghash(&self.ek, &ctr_block, buffer, acc.0, h_polyval, &self.h_powers_rev) }; - acc ^= u128::from_be_bytes(length_block); + acc.0 ^= u128::from_be_bytes(length_block); // SAFETY: POWER8 carryless multiply because backend resolution confirmed POWER8 crypto. - acc = unsafe { polyval::ppc_clmul128_reduce_inline(acc, h_polyval) }; - let tag_bytes = encrypt_j0_tag(&self.ek, &j0, acc); + acc.0 = unsafe { polyval::ppc_clmul128_reduce_inline(acc.0, h_polyval) }; + let tag_bytes = encrypt_j0_tag(&self.ek, &j0, acc.0); return Ok(Aes128GcmTag::from_bytes(tag_bytes)); } @@ -855,18 +866,18 @@ impl Aead for Aes128Gcm { // SAFETY: x86 GHASH AAD path because: // 1. Backend resolution selected X86VaesVpclmul only after CPUID confirmed VPCLMULQDQ. // 2. `aad` is a valid byte slice; padding is handled inside the helper. - let mut acc = unsafe { ghash_update_padded_wide_x86(0, h_polyval, tables.h_powers_rev, aad) }; + let mut acc = GhashAccumulator(unsafe { ghash_update_padded_wide_x86(0, h_polyval, tables.h_powers_rev, aad) }); // SAFETY: fused x86 AES-GCM open because: // 1. Backend resolution selected X86VaesVpclmul only after CPUID confirmed VAES, VPCLMULQDQ, and // AES-NI. // 2. The helper GHASHes ciphertext bytes before decrypting each chunk in place. - acc = unsafe { aes::aes128_ctr32_decrypt_be_wide_ghash(&self.ek, &ctr_block, buffer, acc, tables) }; - acc ^= u128::from_be_bytes(length_block); + acc.0 = unsafe { aes::aes128_ctr32_decrypt_be_wide_ghash(&self.ek, &ctr_block, buffer, acc.0, tables) }; + acc.0 ^= u128::from_be_bytes(length_block); // SAFETY: x86 GHASH final multiply because: // 1. Backend resolution selected X86VaesVpclmul only after CPUID confirmed PCLMULQDQ. // 2. `acc` and `h_polyval` are initialized GHASH field elements. - acc = unsafe { polyval::x86_clmul128_reduce_inline(acc, h_polyval) }; - let expected = encrypt_j0_tag(&self.ek, &j0, acc); + acc.0 = unsafe { polyval::x86_clmul128_reduce_inline(acc.0, h_polyval) }; + let expected = encrypt_j0_tag(&self.ek, &j0, acc.0); if !ct::fixed_eq(&expected, tag.as_bytes()).declassify() { ct::zeroize(buffer); return Err(OpenError::verification()); @@ -880,25 +891,25 @@ impl Aead for Aes128Gcm { && crate::platform::caps().has(crate::platform::caps::x86::PCLMUL_READY) { let h_polyval = self.h_powers_rev[3]; - let mut acc = ghash_update_padded_wide(0, h_polyval, &self.h_powers_rev, aad); + let mut acc = GhashAccumulator(ghash_update_padded_wide(0, h_polyval, &self.h_powers_rev, aad)); // SAFETY: fused x86 AES-NI/PCLMUL open because: // 1. Backend resolution selected X86AesniPclmul only after CPUID confirmed AES-NI and PCLMULQDQ. // 2. The extra PCLMUL_READY check confirms SSSE3 for in-register GHASH byte reversal. // 3. The helper GHASHes ciphertext before decrypting each chunk in place. - acc = unsafe { + acc.0 = unsafe { aes::aes128_ctr32_decrypt_be_aesni_pclmul_ghash( &self.ek, &ctr_block, buffer, - acc, + acc.0, h_polyval, &self.h_powers_rev, ) }; - acc ^= u128::from_be_bytes(length_block); + acc.0 ^= u128::from_be_bytes(length_block); // SAFETY: x86 GHASH final multiply because PCLMUL_READY was checked above. - acc = unsafe { polyval::x86_clmul128_reduce_inline(acc, h_polyval) }; - let expected = encrypt_j0_tag(&self.ek, &j0, acc); + acc.0 = unsafe { polyval::x86_clmul128_reduce_inline(acc.0, h_polyval) }; + let expected = encrypt_j0_tag(&self.ek, &j0, acc.0); if !ct::fixed_eq(&expected, tag.as_bytes()).declassify() { ct::zeroize(buffer); return Err(OpenError::verification()); @@ -916,7 +927,8 @@ impl Aead for Aes128Gcm { // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed AES-CE // and PMULL. // 2. `aad` is a valid byte slice; padding is handled inside the helper. - let mut acc = unsafe { ghash_update_padded_wide_aarch64(0, h_polyval, &self.h_powers_rev, aad) }; + let mut acc = + GhashAccumulator(unsafe { ghash_update_padded_wide_aarch64(0, h_polyval, &self.h_powers_rev, aad) }); // SAFETY: fused intrinsic AArch64 AES-GCM open because: // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed AES-CE // and PMULL. @@ -934,13 +946,13 @@ impl Aead for Aes128Gcm { }; // SAFETY: backend selection confirmed AES-CE/PMULL, and the helper GHASHes ciphertext before // decrypting the valid buffer in place. - acc = unsafe { aes::aes128_ctr32_decrypt_be_aarch64_ghash(&self.ek, &ctr_block, buffer, acc, &tables) }; - acc ^= u128::from_be_bytes(length_block); + acc.0 = unsafe { aes::aes128_ctr32_decrypt_be_aarch64_ghash(&self.ek, &ctr_block, buffer, acc.0, &tables) }; + acc.0 ^= u128::from_be_bytes(length_block); // SAFETY: aarch64 GHASH final multiply because: // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed PMULL. // 2. `acc` and `h_polyval` are initialized GHASH field elements. - acc = unsafe { polyval::aarch64_clmul128_reduce_inline(acc, h_polyval) }; - let expected = encrypt_j0_tag(&self.ek, &j0, acc); + acc.0 = unsafe { polyval::aarch64_clmul128_reduce_inline(acc.0, h_polyval) }; + let expected = encrypt_j0_tag(&self.ek, &j0, acc.0); if !ct::fixed_eq(&expected, tag.as_bytes()).declassify() { ct::zeroize(buffer); return Err(OpenError::verification()); @@ -955,18 +967,18 @@ impl Aead for Aes128Gcm { // 1. Backend resolution selected `Power8Crypto` only after runtime detection confirmed POWER8 // crypto. // 2. `aad` is a valid byte slice; padding is handled inside the helper. - let mut acc = unsafe { ghash_update_padded_wide_ppc(0, h_polyval, &self.h_powers_rev, aad) }; + let mut acc = GhashAccumulator(unsafe { ghash_update_padded_wide_ppc(0, h_polyval, &self.h_powers_rev, aad) }); // SAFETY: fused POWER AES-GCM open because: // 1. Backend resolution selected `Power8Crypto` only after runtime detection confirmed POWER8 // crypto. // 2. The helper GHASHes ciphertext before decrypting the valid buffer in place. - acc = unsafe { - aes::aes128_ctr32_decrypt_be_ppc_ghash(&self.ek, &ctr_block, buffer, acc, h_polyval, &self.h_powers_rev) + acc.0 = unsafe { + aes::aes128_ctr32_decrypt_be_ppc_ghash(&self.ek, &ctr_block, buffer, acc.0, h_polyval, &self.h_powers_rev) }; - acc ^= u128::from_be_bytes(length_block); + acc.0 ^= u128::from_be_bytes(length_block); // SAFETY: POWER8 carryless multiply because backend resolution confirmed POWER8 crypto. - acc = unsafe { polyval::ppc_clmul128_reduce_inline(acc, h_polyval) }; - let expected = encrypt_j0_tag(&self.ek, &j0, acc); + acc.0 = unsafe { polyval::ppc_clmul128_reduce_inline(acc.0, h_polyval) }; + let expected = encrypt_j0_tag(&self.ek, &j0, acc.0); if !ct::fixed_eq(&expected, tag.as_bytes()).declassify() { ct::zeroize(buffer); return Err(OpenError::verification()); diff --git a/src/aead/aes128gcmsiv.rs b/src/aead/aes128gcmsiv.rs index 459a2a2f..2d9cceb4 100644 --- a/src/aead/aes128gcmsiv.rs +++ b/src/aead/aes128gcmsiv.rs @@ -16,9 +16,8 @@ const KEY_SIZE: usize = 16; const TAG_SIZE: usize = 16; const NONCE_SIZE: usize = Nonce96::LENGTH; -/// Maximum plaintext length: 2^36 - 32 bytes (per RFC 8452 §5). -/// Beyond this the 32-bit CTR counter wraps, causing keystream reuse. -const MAX_PLAINTEXT_LEN: u64 = (1u64 << 36).strict_sub(32); +/// Maximum plaintext and additional-data length: 2^36 bytes (RFC 8452 §6). +const MAX_INPUT_LEN: u64 = 1u64 << 36; define_aead_key_type!(Aes128GcmSivKey, KEY_SIZE, "AES-128-GCM-SIV secret key (16 bytes)."); @@ -1404,7 +1403,8 @@ impl Aead for Aes128GcmSiv { buffer: &mut [u8], _token: crate::traits::aead::SealToken, ) -> Result { - super::seal_bounded_length_as_u64(buffer.len(), MAX_PLAINTEXT_LEN)?; + super::seal_bounded_length_as_u64(aad.len(), MAX_INPUT_LEN)?; + super::seal_bounded_length_as_u64(buffer.len(), MAX_INPUT_LEN)?; super::seal_bit_lengths(aad.len(), buffer.len())?; // Wide path: VPCLMULQDQ POLYVAL + VAES-512 CTR when available. @@ -1492,7 +1492,8 @@ impl Aead for Aes128GcmSiv { buffer: &mut [u8], tag: &Self::Tag, ) -> Result<(), OpenError> { - super::open_bounded_length_as_u64(buffer.len(), MAX_PLAINTEXT_LEN)?; + super::open_bounded_length_as_u64(aad.len(), MAX_INPUT_LEN)?; + super::open_bounded_length_as_u64(buffer.len(), MAX_INPUT_LEN)?; super::open_bit_lengths(aad.len(), buffer.len())?; // Wide path: VAES-512 CTR + VPCLMULQDQ POLYVAL when available. @@ -1784,6 +1785,15 @@ mod tests { assert!(Aes128GcmSiv::tag_from_slice(&[0u8; 16]).is_ok()); } + #[test] + #[cfg(target_pointer_width = "64")] + fn aes128gcmsiv_input_limit_matches_rfc8452() { + for len in [MAX_INPUT_LEN.strict_sub(1), MAX_INPUT_LEN] { + assert!(super::super::try_bounded_length_as_u64(len as usize, MAX_INPUT_LEN).is_ok()); + } + assert!(super::super::try_bounded_length_as_u64(MAX_INPUT_LEN.strict_add(1) as usize, MAX_INPUT_LEN).is_err()); + } + // --- Hex helpers --- fn hex16(hex: &str) -> [u8; 16] { diff --git a/src/aead/aes256gcm.rs b/src/aead/aes256gcm.rs index 28af183a..2c806bc6 100644 --- a/src/aead/aes256gcm.rs +++ b/src/aead/aes256gcm.rs @@ -208,6 +208,14 @@ impl Aes256Gcm { // GCM construction internals (NIST SP 800-38D) +struct GhashAccumulator(u128); + +impl Drop for GhashAccumulator { + fn drop(&mut self) { + ct::zeroize_words(core::slice::from_mut(&mut self.0)); + } +} + /// Build the initial counter block J0 and first CTR block for a 96-bit IV. /// /// J0 = IV || 0x00000001 (NIST SP 800-38D § 7.1, when len(IV) = 96). @@ -235,15 +243,15 @@ fn compute_tag( aad: &[u8], ciphertext: &[u8], ) -> Result<[u8; TAG_SIZE], LengthOverflow> { - let mut acc = 0u128; - acc = ghash_update_padded(acc, h_polyval, aad); - acc = ghash_update_padded(acc, h_polyval, ciphertext); + let mut acc = GhashAccumulator(0); + acc.0 = ghash_update_padded(acc.0, h_polyval, aad); + acc.0 = ghash_update_padded(acc.0, h_polyval, ciphertext); let length_block = super::AeadByteLengths::try_new_bit_lengths(aad.len(), ciphertext.len())?.to_be_bits_block(); - acc ^= u128::from_be_bytes(length_block); - acc = polyval::clmul128_reduce(acc, h_polyval); + acc.0 ^= u128::from_be_bytes(length_block); + acc.0 = polyval::clmul128_reduce(acc.0, h_polyval); - Ok(encrypt_j0_tag(ek, j0, acc)) + Ok(encrypt_j0_tag(ek, j0, acc.0)) } #[inline] @@ -356,8 +364,8 @@ fn compute_tag_short_wide( let mut blocks = [0u128; 4]; let start = 4usize.strict_sub(block_count); blocks[start..].copy_from_slice(&sequence[..block_count]); - let acc = polyval::accumulate_4blocks(0, h_polyval, h_powers_rev, &blocks); - Ok(Some(encrypt_j0_tag(ek, j0, acc))) + let acc = GhashAccumulator(polyval::accumulate_4blocks(0, h_polyval, h_powers_rev, &blocks)); + Ok(Some(encrypt_j0_tag(ek, j0, acc.0))) } #[inline(always)] @@ -379,24 +387,26 @@ pub fn diag_aes256gcm_ctr32_be(cipher: &Aes256Gcm, nonce: &Nonce96, plaintext: & } #[cfg(feature = "diag")] +#[unsafe(no_mangle)] +#[inline(never)] #[must_use] pub fn diag_aes256gcm_ghash(cipher: &Aes256Gcm, aad: &[u8], ciphertext: &[u8]) -> [u8; 16] { let h_polyval = cipher.h_powers_rev[3]; - let mut acc = 0u128; + let mut acc = GhashAccumulator(0); if should_use_wide_ghash(cipher.backend, aad.len(), ciphertext.len()) { - acc = ghash_update_padded_wide(acc, h_polyval, &cipher.h_powers_rev, aad); - acc = ghash_update_padded_wide(acc, h_polyval, &cipher.h_powers_rev, ciphertext); + acc.0 = ghash_update_padded_wide(acc.0, h_polyval, &cipher.h_powers_rev, aad); + acc.0 = ghash_update_padded_wide(acc.0, h_polyval, &cipher.h_powers_rev, ciphertext); } else { - acc = ghash_update_padded(acc, h_polyval, aad); - acc = ghash_update_padded(acc, h_polyval, ciphertext); + acc.0 = ghash_update_padded(acc.0, h_polyval, aad); + acc.0 = ghash_update_padded(acc.0, h_polyval, ciphertext); } let length_block = match super::AeadByteLengths::try_new_bit_lengths(aad.len(), ciphertext.len()) { Ok(lengths) => lengths.to_be_bits_block(), Err(_) => return [0u8; 16], }; - acc ^= u128::from_be_bytes(length_block); - acc = polyval::clmul128_reduce(acc, h_polyval); - acc.to_be_bytes() + acc.0 ^= u128::from_be_bytes(length_block); + acc.0 = polyval::clmul128_reduce(acc.0, h_polyval); + acc.0.to_be_bytes() } #[cfg(feature = "diag")] @@ -434,16 +444,16 @@ fn compute_tag_wide( aad: &[u8], ciphertext: &[u8], ) -> Result<[u8; TAG_SIZE], LengthOverflow> { - let mut acc = 0u128; - acc = ghash_update_padded_wide(acc, h_polyval, h_powers_rev, aad); - acc = ghash_update_padded_wide(acc, h_polyval, h_powers_rev, ciphertext); + let mut acc = GhashAccumulator(0); + acc.0 = ghash_update_padded_wide(acc.0, h_polyval, h_powers_rev, aad); + acc.0 = ghash_update_padded_wide(acc.0, h_polyval, h_powers_rev, ciphertext); // Length block. let length_block = super::AeadByteLengths::try_new_bit_lengths(aad.len(), ciphertext.len())?.to_be_bits_block(); - acc ^= u128::from_be_bytes(length_block); - acc = polyval::clmul128_reduce(acc, h_polyval); + acc.0 ^= u128::from_be_bytes(length_block); + acc.0 = polyval::clmul128_reduce(acc.0, h_polyval); - Ok(encrypt_j0_tag(ek, j0, acc)) + Ok(encrypt_j0_tag(ek, j0, acc.0)) } /// Update GHASH using VPCLMUL without scalar block packing. @@ -713,18 +723,18 @@ impl Aead for Aes256Gcm { // SAFETY: x86 GHASH AAD path because: // 1. Backend resolution selected X86VaesVpclmul only after CPUID confirmed VPCLMULQDQ. // 2. `aad` is a valid byte slice; padding is handled inside the helper. - let mut acc = unsafe { ghash_update_padded_wide_x86(0, h_polyval, tables.h_powers_rev, aad) }; + let mut acc = GhashAccumulator(unsafe { ghash_update_padded_wide_x86(0, h_polyval, tables.h_powers_rev, aad) }); // SAFETY: fused x86 AES-GCM sealing because: // 1. Backend resolution selected X86VaesVpclmul only after CPUID confirmed VAES, VPCLMULQDQ, and // AES-NI. // 2. The helper encrypts `buffer` in place and folds the resulting ciphertext into GHASH. - acc = unsafe { aes::aes256_ctr32_encrypt_be_wide_ghash(&self.ek, &ctr_block, buffer, acc, tables) }; - acc ^= u128::from_be_bytes(length_block); + acc.0 = unsafe { aes::aes256_ctr32_encrypt_be_wide_ghash(&self.ek, &ctr_block, buffer, acc.0, tables) }; + acc.0 ^= u128::from_be_bytes(length_block); // SAFETY: x86 GHASH final multiply because: // 1. Backend resolution selected X86VaesVpclmul only after CPUID confirmed PCLMULQDQ. // 2. `acc` and `h_polyval` are initialized GHASH field elements. - acc = unsafe { polyval::x86_clmul128_reduce_inline(acc, h_polyval) }; - let tag_bytes = encrypt_j0_tag(&self.ek, &j0, acc); + acc.0 = unsafe { polyval::x86_clmul128_reduce_inline(acc.0, h_polyval) }; + let tag_bytes = encrypt_j0_tag(&self.ek, &j0, acc.0); return Ok(Aes256GcmTag::from_bytes(tag_bytes)); } @@ -734,25 +744,25 @@ impl Aead for Aes256Gcm { && crate::platform::caps().has(crate::platform::caps::x86::PCLMUL_READY) { let h_polyval = self.h_powers_rev[3]; - let mut acc = ghash_update_padded_wide(0, h_polyval, &self.h_powers_rev, aad); + let mut acc = GhashAccumulator(ghash_update_padded_wide(0, h_polyval, &self.h_powers_rev, aad)); // SAFETY: fused x86 AES-NI/PCLMUL sealing because: // 1. Backend resolution selected X86AesniPclmul only after CPUID confirmed AES-NI and PCLMULQDQ. // 2. The extra PCLMUL_READY check confirms SSSE3 for in-register GHASH byte reversal. // 3. The helper encrypts `buffer` in place and folds ciphertext into GHASH. - acc = unsafe { + acc.0 = unsafe { aes::aes256_ctr32_encrypt_be_aesni_pclmul_ghash( &self.ek, &ctr_block, buffer, - acc, + acc.0, h_polyval, &self.h_powers_rev, ) }; - acc ^= u128::from_be_bytes(length_block); + acc.0 ^= u128::from_be_bytes(length_block); // SAFETY: x86 GHASH final multiply because PCLMUL_READY was checked above. - acc = unsafe { polyval::x86_clmul128_reduce_inline(acc, h_polyval) }; - let tag_bytes = encrypt_j0_tag(&self.ek, &j0, acc); + acc.0 = unsafe { polyval::x86_clmul128_reduce_inline(acc.0, h_polyval) }; + let tag_bytes = encrypt_j0_tag(&self.ek, &j0, acc.0); return Ok(Aes256GcmTag::from_bytes(tag_bytes)); } @@ -766,7 +776,8 @@ impl Aead for Aes256Gcm { // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed AES-CE // and PMULL. // 2. `aad` is a valid byte slice; padding is handled inside the helper. - let mut acc = unsafe { ghash_update_padded_wide_aarch64(0, h_polyval, &self.h_powers_rev, aad) }; + let mut acc = + GhashAccumulator(unsafe { ghash_update_padded_wide_aarch64(0, h_polyval, &self.h_powers_rev, aad) }); // SAFETY: fused intrinsic AArch64 AES-GCM sealing because: // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed AES-CE // and PMULL. @@ -784,13 +795,13 @@ impl Aead for Aes256Gcm { }; // SAFETY: backend selection confirmed AES-CE/PMULL, and the helper encrypts the valid buffer // in place while folding ciphertext into GHASH. - acc = unsafe { aes::aes256_ctr32_encrypt_be_aarch64_ghash(&self.ek, &ctr_block, buffer, acc, &tables) }; - acc ^= u128::from_be_bytes(length_block); + acc.0 = unsafe { aes::aes256_ctr32_encrypt_be_aarch64_ghash(&self.ek, &ctr_block, buffer, acc.0, &tables) }; + acc.0 ^= u128::from_be_bytes(length_block); // SAFETY: aarch64 GHASH final multiply because: // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed PMULL. // 2. `acc` and `h_polyval` are initialized GHASH field elements. - acc = unsafe { polyval::aarch64_clmul128_reduce_inline(acc, h_polyval) }; - let tag_bytes = encrypt_j0_tag(&self.ek, &j0, acc); + acc.0 = unsafe { polyval::aarch64_clmul128_reduce_inline(acc.0, h_polyval) }; + let tag_bytes = encrypt_j0_tag(&self.ek, &j0, acc.0); return Ok(Aes256GcmTag::from_bytes(tag_bytes)); } @@ -801,18 +812,18 @@ impl Aead for Aes256Gcm { // 1. Backend resolution selected `Power8Crypto` only after runtime detection confirmed POWER8 // crypto. // 2. `aad` is a valid byte slice; padding is handled inside the helper. - let mut acc = unsafe { ghash_update_padded_wide_ppc(0, h_polyval, &self.h_powers_rev, aad) }; + let mut acc = GhashAccumulator(unsafe { ghash_update_padded_wide_ppc(0, h_polyval, &self.h_powers_rev, aad) }); // SAFETY: fused POWER AES-GCM sealing because: // 1. Backend resolution selected `Power8Crypto` only after runtime detection confirmed POWER8 // crypto. // 2. The helper encrypts `buffer` in place and folds ciphertext into GHASH. - acc = unsafe { - aes::aes256_ctr32_encrypt_be_ppc_ghash(&self.ek, &ctr_block, buffer, acc, h_polyval, &self.h_powers_rev) + acc.0 = unsafe { + aes::aes256_ctr32_encrypt_be_ppc_ghash(&self.ek, &ctr_block, buffer, acc.0, h_polyval, &self.h_powers_rev) }; - acc ^= u128::from_be_bytes(length_block); + acc.0 ^= u128::from_be_bytes(length_block); // SAFETY: POWER8 carryless multiply because backend resolution confirmed POWER8 crypto. - acc = unsafe { polyval::ppc_clmul128_reduce_inline(acc, h_polyval) }; - let tag_bytes = encrypt_j0_tag(&self.ek, &j0, acc); + acc.0 = unsafe { polyval::ppc_clmul128_reduce_inline(acc.0, h_polyval) }; + let tag_bytes = encrypt_j0_tag(&self.ek, &j0, acc.0); return Ok(Aes256GcmTag::from_bytes(tag_bytes)); } @@ -858,18 +869,18 @@ impl Aead for Aes256Gcm { // SAFETY: x86 GHASH AAD path because: // 1. Backend resolution selected X86VaesVpclmul only after CPUID confirmed VPCLMULQDQ. // 2. `aad` is a valid byte slice; padding is handled inside the helper. - let mut acc = unsafe { ghash_update_padded_wide_x86(0, h_polyval, tables.h_powers_rev, aad) }; + let mut acc = GhashAccumulator(unsafe { ghash_update_padded_wide_x86(0, h_polyval, tables.h_powers_rev, aad) }); // SAFETY: fused x86 AES-GCM open because: // 1. Backend resolution selected X86VaesVpclmul only after CPUID confirmed VAES, VPCLMULQDQ, and // AES-NI. // 2. The helper GHASHes ciphertext bytes before decrypting each chunk in place. - acc = unsafe { aes::aes256_ctr32_decrypt_be_wide_ghash(&self.ek, &ctr_block, buffer, acc, tables) }; - acc ^= u128::from_be_bytes(length_block); + acc.0 = unsafe { aes::aes256_ctr32_decrypt_be_wide_ghash(&self.ek, &ctr_block, buffer, acc.0, tables) }; + acc.0 ^= u128::from_be_bytes(length_block); // SAFETY: x86 GHASH final multiply because: // 1. Backend resolution selected X86VaesVpclmul only after CPUID confirmed PCLMULQDQ. // 2. `acc` and `h_polyval` are initialized GHASH field elements. - acc = unsafe { polyval::x86_clmul128_reduce_inline(acc, h_polyval) }; - let expected = encrypt_j0_tag(&self.ek, &j0, acc); + acc.0 = unsafe { polyval::x86_clmul128_reduce_inline(acc.0, h_polyval) }; + let expected = encrypt_j0_tag(&self.ek, &j0, acc.0); if !ct::fixed_eq(&expected, tag.as_bytes()).declassify() { ct::zeroize(buffer); return Err(OpenError::verification()); @@ -883,25 +894,25 @@ impl Aead for Aes256Gcm { && crate::platform::caps().has(crate::platform::caps::x86::PCLMUL_READY) { let h_polyval = self.h_powers_rev[3]; - let mut acc = ghash_update_padded_wide(0, h_polyval, &self.h_powers_rev, aad); + let mut acc = GhashAccumulator(ghash_update_padded_wide(0, h_polyval, &self.h_powers_rev, aad)); // SAFETY: fused x86 AES-NI/PCLMUL open because: // 1. Backend resolution selected X86AesniPclmul only after CPUID confirmed AES-NI and PCLMULQDQ. // 2. The extra PCLMUL_READY check confirms SSSE3 for in-register GHASH byte reversal. // 3. The helper GHASHes ciphertext before decrypting each chunk in place. - acc = unsafe { + acc.0 = unsafe { aes::aes256_ctr32_decrypt_be_aesni_pclmul_ghash( &self.ek, &ctr_block, buffer, - acc, + acc.0, h_polyval, &self.h_powers_rev, ) }; - acc ^= u128::from_be_bytes(length_block); + acc.0 ^= u128::from_be_bytes(length_block); // SAFETY: x86 GHASH final multiply because PCLMUL_READY was checked above. - acc = unsafe { polyval::x86_clmul128_reduce_inline(acc, h_polyval) }; - let expected = encrypt_j0_tag(&self.ek, &j0, acc); + acc.0 = unsafe { polyval::x86_clmul128_reduce_inline(acc.0, h_polyval) }; + let expected = encrypt_j0_tag(&self.ek, &j0, acc.0); if !ct::fixed_eq(&expected, tag.as_bytes()).declassify() { ct::zeroize(buffer); return Err(OpenError::verification()); @@ -919,7 +930,8 @@ impl Aead for Aes256Gcm { // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed AES-CE // and PMULL. // 2. `aad` is a valid byte slice; padding is handled inside the helper. - let mut acc = unsafe { ghash_update_padded_wide_aarch64(0, h_polyval, &self.h_powers_rev, aad) }; + let mut acc = + GhashAccumulator(unsafe { ghash_update_padded_wide_aarch64(0, h_polyval, &self.h_powers_rev, aad) }); // SAFETY: fused intrinsic AArch64 AES-GCM open because: // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed AES-CE // and PMULL. @@ -937,13 +949,13 @@ impl Aead for Aes256Gcm { }; // SAFETY: backend selection confirmed AES-CE/PMULL, and the helper GHASHes ciphertext before // decrypting the valid buffer in place. - acc = unsafe { aes::aes256_ctr32_decrypt_be_aarch64_ghash(&self.ek, &ctr_block, buffer, acc, &tables) }; - acc ^= u128::from_be_bytes(length_block); + acc.0 = unsafe { aes::aes256_ctr32_decrypt_be_aarch64_ghash(&self.ek, &ctr_block, buffer, acc.0, &tables) }; + acc.0 ^= u128::from_be_bytes(length_block); // SAFETY: aarch64 GHASH final multiply because: // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed PMULL. // 2. `acc` and `h_polyval` are initialized GHASH field elements. - acc = unsafe { polyval::aarch64_clmul128_reduce_inline(acc, h_polyval) }; - let expected = encrypt_j0_tag(&self.ek, &j0, acc); + acc.0 = unsafe { polyval::aarch64_clmul128_reduce_inline(acc.0, h_polyval) }; + let expected = encrypt_j0_tag(&self.ek, &j0, acc.0); if !ct::fixed_eq(&expected, tag.as_bytes()).declassify() { ct::zeroize(buffer); return Err(OpenError::verification()); @@ -958,18 +970,18 @@ impl Aead for Aes256Gcm { // 1. Backend resolution selected `Power8Crypto` only after runtime detection confirmed POWER8 // crypto. // 2. `aad` is a valid byte slice; padding is handled inside the helper. - let mut acc = unsafe { ghash_update_padded_wide_ppc(0, h_polyval, &self.h_powers_rev, aad) }; + let mut acc = GhashAccumulator(unsafe { ghash_update_padded_wide_ppc(0, h_polyval, &self.h_powers_rev, aad) }); // SAFETY: fused POWER AES-GCM open because: // 1. Backend resolution selected `Power8Crypto` only after runtime detection confirmed POWER8 // crypto. // 2. The helper GHASHes ciphertext before decrypting the valid buffer in place. - acc = unsafe { - aes::aes256_ctr32_decrypt_be_ppc_ghash(&self.ek, &ctr_block, buffer, acc, h_polyval, &self.h_powers_rev) + acc.0 = unsafe { + aes::aes256_ctr32_decrypt_be_ppc_ghash(&self.ek, &ctr_block, buffer, acc.0, h_polyval, &self.h_powers_rev) }; - acc ^= u128::from_be_bytes(length_block); + acc.0 ^= u128::from_be_bytes(length_block); // SAFETY: POWER8 carryless multiply because backend resolution confirmed POWER8 crypto. - acc = unsafe { polyval::ppc_clmul128_reduce_inline(acc, h_polyval) }; - let expected = encrypt_j0_tag(&self.ek, &j0, acc); + acc.0 = unsafe { polyval::ppc_clmul128_reduce_inline(acc.0, h_polyval) }; + let expected = encrypt_j0_tag(&self.ek, &j0, acc.0); if !ct::fixed_eq(&expected, tag.as_bytes()).declassify() { ct::zeroize(buffer); return Err(OpenError::verification()); diff --git a/src/aead/aes256gcmsiv.rs b/src/aead/aes256gcmsiv.rs index 87baa773..57824a7f 100644 --- a/src/aead/aes256gcmsiv.rs +++ b/src/aead/aes256gcmsiv.rs @@ -16,9 +16,8 @@ const KEY_SIZE: usize = 32; const TAG_SIZE: usize = 16; const NONCE_SIZE: usize = Nonce96::LENGTH; -/// Maximum plaintext length: 2^36 - 32 bytes (per RFC 8452 §5). -/// Beyond this the 32-bit CTR counter wraps, causing keystream reuse. -const MAX_PLAINTEXT_LEN: u64 = (1u64 << 36).strict_sub(32); +/// Maximum plaintext and additional-data length: 2^36 bytes (RFC 8452 §6). +const MAX_INPUT_LEN: u64 = 1u64 << 36; define_aead_key_type!(Aes256GcmSivKey, KEY_SIZE, "AES-256-GCM-SIV secret key (32 bytes)."); @@ -1471,7 +1470,8 @@ impl Aead for Aes256GcmSiv { buffer: &mut [u8], _token: crate::traits::aead::SealToken, ) -> Result { - super::seal_bounded_length_as_u64(buffer.len(), MAX_PLAINTEXT_LEN)?; + super::seal_bounded_length_as_u64(aad.len(), MAX_INPUT_LEN)?; + super::seal_bounded_length_as_u64(buffer.len(), MAX_INPUT_LEN)?; super::seal_bit_lengths(aad.len(), buffer.len())?; // Wide path: VPCLMULQDQ POLYVAL + VAES-512 CTR when available. @@ -1559,7 +1559,8 @@ impl Aead for Aes256GcmSiv { buffer: &mut [u8], tag: &Self::Tag, ) -> Result<(), OpenError> { - super::open_bounded_length_as_u64(buffer.len(), MAX_PLAINTEXT_LEN)?; + super::open_bounded_length_as_u64(aad.len(), MAX_INPUT_LEN)?; + super::open_bounded_length_as_u64(buffer.len(), MAX_INPUT_LEN)?; super::open_bit_lengths(aad.len(), buffer.len())?; // Wide path: VAES-512 CTR + VPCLMULQDQ POLYVAL when available. @@ -1853,6 +1854,15 @@ mod tests { assert!(Aes256GcmSiv::tag_from_slice(&[0u8; 16]).is_ok()); } + #[test] + #[cfg(target_pointer_width = "64")] + fn aes256gcmsiv_input_limit_matches_rfc8452() { + for len in [MAX_INPUT_LEN.strict_sub(1), MAX_INPUT_LEN] { + assert!(super::super::try_bounded_length_as_u64(len as usize, MAX_INPUT_LEN).is_ok()); + } + assert!(super::super::try_bounded_length_as_u64(MAX_INPUT_LEN.strict_add(1) as usize, MAX_INPUT_LEN).is_err()); + } + /// RFC 8452 Appendix C.2 vector with a different key (empty PT+AAD). #[test] fn aes256gcmsiv_different_key_vector() { diff --git a/src/aead/ascon128.rs b/src/aead/ascon128.rs index dee84745..51cad04a 100644 --- a/src/aead/ascon128.rs +++ b/src/aead/ascon128.rs @@ -313,7 +313,9 @@ impl Aead for AsconAead128 { tail.copy_from_slice(&s[sidx].to_le_bytes()[..tail.len()]); } - Ok(AsconAead128Tag::from_bytes(self.finalize(&mut s))) + let tag = self.finalize(&mut s); + ct::zeroize_words(&mut s); + Ok(AsconAead128Tag::from_bytes(tag)) } fn decrypt_in_place( @@ -356,6 +358,7 @@ impl Aead for AsconAead128 { } let expected = self.finalize(&mut s); + ct::zeroize_words(&mut s); if !ct::fixed_eq(&expected, tag.as_bytes()).declassify() { ct::zeroize(buffer); return Err(OpenError::verification()); @@ -366,6 +369,8 @@ impl Aead for AsconAead128 { } #[cfg(feature = "diag")] +#[unsafe(no_mangle)] +#[inline(never)] pub fn diag_ascon_aead128_tag_portable( key: &[u8; KEY_SIZE], nonce: &[u8; NONCE_SIZE], @@ -381,6 +386,7 @@ pub fn diag_ascon_aead128_tag_portable( s[1] ^= load_bytes(&block[8..]); permute_8_portable(&mut s); let tag = cipher.finalize(&mut s); + ct::zeroize_words(&mut s); ct::fixed_eq(&tag, expected) } diff --git a/src/aead/chacha20poly1305.rs b/src/aead/chacha20poly1305.rs index ad96c186..0309998f 100644 --- a/src/aead/chacha20poly1305.rs +++ b/src/aead/chacha20poly1305.rs @@ -269,11 +269,8 @@ impl ChaCha20Poly1305 { return false; } - // Measured on the 2026-07-01 x86_64 Linux bench run: - // - Sapphire Rapids regresses beyond 256 bytes. - // - AMD Zen5 regresses beyond 1024 bytes. - // - The remaining measured AVX2+BMI2 lanes, Zen4 and Ice Lake, benefit from the integrated pass - // shape across the sampled non-empty sizes. + // The manually maintained x86 policy bounds the integrated assembly path + // on the two microarchitecture classes with explicit crossover limits. if caps.has(x86::INTEL_SAPPHIRE_RAPIDS) { plaintext_len <= X86_64_ASM_SPR_MAX } else if caps.has(x86::AMD_ZEN5) { @@ -296,10 +293,9 @@ impl ChaCha20Poly1305 { return false; } - // Measured on the 2026-07-01 decrypt bench: - // - all sampled x86_64 CPUs lose 1..=256 bytes on the generic split open path. - // - AMD Zen4 loses through the full sampled matrix; the repo only has a Zen5-specific AMD - // discriminator, so AMD without Zen5 takes integrated open for every non-empty size. + // AMD without the Zen5 discriminator keeps the integrated open path for + // every non-empty size; other x86 classes use the configured short-input + // boundary. if caps.has(x86::AMD) && !caps.has(x86::AMD_ZEN5) { true } else { @@ -846,7 +842,7 @@ mod tests { #[cfg(all(target_arch = "x86_64", target_os = "linux"))] #[test] - fn x86_64_asm_policy_matches_measured_thresholds() { + fn x86_64_asm_policy_matches_configured_thresholds() { use crate::platform::{Caps, caps::x86}; let avx2_bmi2 = x86::AVX2 | x86::BMI2; @@ -917,13 +913,13 @@ mod tests { if aad_len > SMALL_AAD_FAST_MAX { assert!( actual.is_none(), - "empty decrypt fast path applied outside its measured gate: aad_len={aad_len}" + "empty decrypt fast path applied outside its configured gate: aad_len={aad_len}" ); continue; } actual - .expect("empty decrypt fast path must apply inside its measured gate") + .expect("empty decrypt fast path must apply inside its configured gate") .unwrap(); let mut bad_tag = expected_tag.to_bytes(); @@ -931,7 +927,7 @@ mod tests { assert_eq!( cipher .decrypt_empty_text_fast(&nonce, &aad, &ChaCha20Poly1305Tag::from_bytes(bad_tag)) - .expect("empty decrypt fast path must apply inside its measured gate"), + .expect("empty decrypt fast path must apply inside its configured gate"), Err(OpenError::verification()) ); } @@ -1002,7 +998,8 @@ mod tests { if plaintext_len == 0 || plaintext_len > POWER_SHORT_FAST_MAX || aad_len > SMALL_AAD_FAST_MAX { assert!( actual_tag.is_none(), - "Power short fast path applied outside its measured gate: plaintext_len={plaintext_len} aad_len={aad_len}" + "Power short fast path applied outside its configured gate: plaintext_len={plaintext_len} \ + aad_len={aad_len}" ); assert_eq!(actual, plaintext); continue; @@ -1013,7 +1010,7 @@ mod tests { .encrypt_in_place_owned_unchecked(&nonce, &aad, &mut expected) .unwrap(); let actual_tag = actual_tag - .expect("Power short fast path must apply inside its measured gate") + .expect("Power short fast path must apply inside its configured gate") .unwrap(); assert_eq!( @@ -1055,7 +1052,7 @@ mod tests { if plaintext_len == 0 || plaintext_len > POWER_SHORT_FAST_MAX || aad_len > SMALL_AAD_FAST_MAX { assert!( actual_result.is_none(), - "Power short decrypt fast path applied outside its measured gate: plaintext_len={plaintext_len} \ + "Power short decrypt fast path applied outside its configured gate: plaintext_len={plaintext_len} \ aad_len={aad_len}" ); assert_eq!(actual, ciphertext); @@ -1063,7 +1060,7 @@ mod tests { } actual_result - .expect("Power short decrypt fast path must apply inside its measured gate") + .expect("Power short decrypt fast path must apply inside its configured gate") .unwrap(); assert_eq!( actual, plaintext, @@ -1076,7 +1073,7 @@ mod tests { assert_eq!( cipher .decrypt_short_text_power_fast(&nonce, &aad, &mut rejected, &ChaCha20Poly1305Tag::from_bytes(bad_tag),) - .expect("Power short decrypt fast path must apply inside its measured gate"), + .expect("Power short decrypt fast path must apply inside its configured gate"), Err(OpenError::verification()) ); assert!( diff --git a/src/aead/poly1305.rs b/src/aead/poly1305.rs index 9afbc80c..557d52f1 100644 --- a/src/aead/poly1305.rs +++ b/src/aead/poly1305.rs @@ -517,6 +517,15 @@ struct State { pad: [u32; 4], } +impl Drop for State { + fn drop(&mut self) { + ct::zeroize_words_no_fence(&mut self.r); + ct::zeroize_words_no_fence(&mut self.h); + ct::zeroize_words_no_fence(&mut self.pad); + core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); + } +} + impl State { #[inline] fn new(key: &[u8; 32]) -> Self { @@ -839,7 +848,8 @@ pub fn diag_chacha20poly1305_authenticate_aead(aad: &[u8], ciphertext: &[u8], ke } #[cfg(feature = "diag")] -#[inline(always)] +#[unsafe(no_mangle)] +#[inline(never)] pub fn diag_poly1305_block_portable_digest(key: &[u8; 32], block: &[u8; 16], partial: bool) -> [u8; 16] { let mut state = State::new(key); state.compute_block_portable(block, partial); diff --git a/src/aead/polyval.rs b/src/aead/polyval.rs index ceef49f1..f6f65c08 100644 --- a/src/aead/polyval.rs +++ b/src/aead/polyval.rs @@ -899,9 +899,8 @@ mod rv_scalar_clmul { /// Combined 128×128 carryless multiply + Montgomery reduce using scalar Zbc. /// - /// Karatsuba decomposition with 3 × clmul/clmulh pairs (6 instructions), - /// then portable Montgomery reduction. ~100x faster than Pornin bmul64 on - /// hardware with Zbc (e.g. SpacemiT K1). + /// Karatsuba decomposition with 3 × clmul/clmulh pairs, followed by portable + /// Montgomery reduction. /// /// # Safety /// Caller must ensure Zbc or Zbkc scalar extension is available. diff --git a/src/aead/targets.rs b/src/aead/targets.rs index 68f6f651..709e29d9 100644 --- a/src/aead/targets.rs +++ b/src/aead/targets.rs @@ -53,8 +53,8 @@ pub enum AeadBackend { Riscv64VectorCrypto, Riscv64Vector, /// Hamburg vperm AES via register-only `vrgather.vv` operations. - /// Kept as an explicit backend, but not selected for V-only RISC-V until - /// it beats the scalar portable path on benchmark hardware. + /// Kept as an explicit backend, but not selected for V-only RISC-V without + /// target-native evidence supporting that dispatch policy. Riscv64Vperm, } @@ -89,11 +89,10 @@ impl AeadBackend { } } -/// Select the best backend class allowed by the detected architecture and caps. +/// Select the backend class allowed by the detected architecture, capabilities, +/// and current dispatch policy. /// -/// This function encodes current dispatch policy, not benchmark fantasy. -/// Unmeasured or unimplemented SIMD classes deliberately resolve to `portable` -/// instead of lying. +/// SIMD classes without accepted target-native evidence resolve to `portable`. #[allow(dead_code)] // Some leaf AEAD builds compile target policy without live dispatch on the host. #[must_use] pub fn select_backend(primitive: AeadPrimitive, arch: Arch, caps: Caps) -> AeadBackend { @@ -205,9 +204,8 @@ fn select_gcm_backend(arch: Arch, caps: Caps) -> AeadBackend { } else if caps.has(riscv::ZKNE) && (caps.has(riscv::ZBC) || caps.has(riscv::ZBKC)) { AeadBackend::Riscv64ScalarCrypto } else { - // Table-free scalar fallback. V-only Hamburg vperm is currently - // much slower than the portable path on RISE benchmark hardware, so - // do not select it automatically. + // V-only Hamburg vperm remains available for forced diagnostics but + // lacks accepted target-native evidence for automatic selection. AeadBackend::Portable } } @@ -228,9 +226,9 @@ const fn select_ascon_backend(arch: Arch) -> AeadBackend { #[inline] #[allow(dead_code)] // Only used when AEGIS-256 needs target-policy dispatch on this target. fn select_aegis_backend(arch: Arch, caps: Caps) -> AeadBackend { - // VAES-256 is intentionally not used for AEGIS-256. The serial update chain - // (6 dependent AES rounds per block) makes cross-lane shuffle overhead in - // the VAES-256 path slower than straight AES-NI. See aegis256.rs encrypt_in_place. + // VAES-256 is intentionally not used for AEGIS-256. Its six dependent state + // lanes require cross-lane shuffles in the packed representation; see the + // XMM-state path in aegis256.rs. match arch { Arch::X86_64 => { if caps.has(x86::AESNI) { @@ -268,9 +266,8 @@ fn select_aegis_backend(arch: Arch, caps: Caps) -> AeadBackend { } else if caps.has(riscv::ZKNE) { AeadBackend::Riscv64ScalarCrypto } else { - // Table-free scalar fallback. V-only Hamburg vperm is currently - // much slower than the portable path on RISE benchmark hardware, so - // do not select it automatically. + // V-only Hamburg vperm remains available for forced diagnostics but + // lacks accepted target-native evidence for automatic selection. AeadBackend::Portable } } @@ -484,8 +481,8 @@ mod tests { AeadBackend::Riscv64ScalarCrypto ); - // Tier 3: V-only falls back to portable until the vperm AES backend is - // measured faster than the scalar portable path on benchmark hardware. + // Tier 3: V-only falls back to portable until target-native evidence + // supports automatic vperm selection. assert_eq!( select_backend(AeadPrimitive::Aes256Gcm, Arch::Riscv64, riscv::V | riscv::ZBC), AeadBackend::Portable @@ -567,7 +564,7 @@ mod tests { } #[test] - fn ascon_stays_portable_until_measured_simd_is_proven() { + fn ascon_stays_portable_until_simd_policy_is_accepted() { assert_eq!( select_backend(AeadPrimitive::AsconAead128, Arch::X86_64, x86::AVX2 | x86::VAES_READY), AeadBackend::Portable diff --git a/src/auth/argon2/dispatch.rs b/src/auth/argon2/dispatch.rs index fb5b1117..1df34ae5 100644 --- a/src/auth/argon2/dispatch.rs +++ b/src/auth/argon2/dispatch.rs @@ -1,7 +1,7 @@ //! Argon2 BlaMka compression kernel dispatch. //! //! Exposes the [`KernelId`] enum and the runtime selector that picks the -//! highest-throughput kernel whose required caps are present on the host. +//! first configured kernel whose required caps are present on the host. //! //! The dispatcher is invoked once per [`super::argon2_hash`] call and the //! resulting [`kernels::CompressFn`] pointer is threaded down through the @@ -177,9 +177,9 @@ static ACTIVE_KERNEL: OnceCache = OnceCache::new(); /// on one 128-byte block. Argon2 BlaMka is a *different* shape — each /// compression is 1024 bytes with 16 P-rounds of 4 independent GBs, so /// a 4-way SIMD kernel has real parallelism to extract even on M-series. -/// The gate on this primitive is measured, not inherited: the NEON kernel -/// is the active one on all aarch64 targets unless a future measurement -/// flips the polarity for a specific host. +/// Current policy selects the NEON kernel on all AArch64 targets. Changing that +/// policy requires representative target-native evidence for this Argon2 +/// workload rather than inference from a different primitive. /// /// # Gate: x86_64 /// @@ -275,7 +275,7 @@ mod tests { fn x86_kernels_ordered_avx512_then_avx2_then_portable() { // The dispatcher walks ALL_KERNELS in order and picks the first whose // caps are present on the host. AVX-512 must precede AVX2 must precede - // portable so the highest-throughput available kernel wins. + // portable so the configured preference order is preserved. let avx512_pos = ALL_KERNELS .iter() .position(|&k| k == KernelId::X86Avx512) diff --git a/src/auth/argon2/mod.rs b/src/auth/argon2/mod.rs index 8616c0a6..3d1c978c 100644 --- a/src/auth/argon2/mod.rs +++ b/src/auth/argon2/mod.rs @@ -2,9 +2,8 @@ //! //! Ships all three variants: //! -//! - [`Argon2d`] — data-dependent indexing, highest throughput, vulnerable to side-channel timing -//! attacks. Useful for non-interactive, trusted- hardware settings (e.g. cryptocurrency -//! proof-of-work). +//! - [`Argon2d`] — data-dependent indexing with corresponding cache-timing leakage. Useful for +//! non-interactive, trusted-hardware settings (e.g. cryptocurrency proof-of-work). //! - [`Argon2i`] — data-independent indexing: memory-access patterns do not depend on the password. //! Useful when the adversary can observe cache or memory-access timing. //! - [`Argon2id`] — hybrid: first half of the first pass runs Argon2i, the rest runs Argon2d. @@ -13,9 +12,9 @@ //! //! The implementation uses the BlaMka compression function from RFC 9106 //! §3.6 layered on top of [`crate::Blake2b`]. A runtime-cached [`KernelId`] -//! dispatcher picks the highest-throughput BlaMka kernel for the host: a -//! 4-way NEON kernel on aarch64 ships today, with per-arch x86_64 / VSX / -//! s390x / RVV / simd128 kernels rolling in behind the same contract. +//! dispatcher selects among the portable implementation and the compiled +//! x86_64, aarch64, POWER, s390x, RISC-V, or wasm32 backends when their +//! required capabilities are available. //! //! # Examples //! @@ -217,13 +216,13 @@ pub enum Argon2Error { SecretTooLong, /// Optional associated-data length exceeds 2^32-1 bytes. AssociatedDataTooLong, + /// The platform entropy source failed while generating a PHC salt. + #[cfg(all(feature = "phc-strings", feature = "getrandom"))] + EntropyUnavailable, /// The requested memory matrix exceeds the target's address space. ResourceOverflow, /// The allocator refused to provide the memory matrix. AllocationFailed, - /// The platform entropy source failed while generating a PHC salt. - #[cfg(all(feature = "phc-strings", feature = "getrandom"))] - EntropyUnavailable, /// Password generation parameters exceed the verifier's resource limits. #[cfg(feature = "phc-strings")] VerificationLimitTooLow, @@ -1212,10 +1211,10 @@ impl Drop for Matrix { /// borrow, the matrix is single-threaded for the call's duration, and the view is dropped before /// control returns to `argon2_hash`. Exclusivity is trivial. /// - **Parallel path** (`fill_slice_parallel`): the view is shared across `rayon::scope` tasks. -/// Disjointness across tasks is upheld by the Argon2 reference-index function (RFC 9106 §3.4), -/// which guarantees that within a single slice every lane writes only to its own segment range -/// and reads only from blocks in already-completed slices (or strictly earlier columns of the -/// same segment, which are this task's own writes). +/// Within a slice, every task writes only its own lane's current segment. Cross-lane references +/// exclude the current slice, while same-lane references may read stable blocks written earlier +/// by the same task. The previous and reference inputs may name the same immutable block, but +/// neither may name the current mutable output block. /// /// `Send + Sync` are required to cross rayon thread boundaries; both are /// `unsafe impl`d on the assumption that callers honour the discipline @@ -1229,10 +1228,10 @@ struct MatrixView { #[cfg(feature = "parallel")] // SAFETY: `MatrixView` is `Send + Sync` only because the parallel filler -// (`fill_slice_parallel`) honours the RFC 9106 §3.4 disjointness discipline: -// 1. Within one slice, lane `i` writes exclusively to its own segment and reads only from (a) -// blocks in already-completed slices, or (b) earlier columns of the in-progress segment which -// the same task wrote. +// (`fill_slice_parallel`) honours the RFC 9106 §3.4 access discipline: +// 1. Within one slice, lane `i` writes exclusively to its own segment. Cross-lane reads exclude +// the current slice; same-lane reads may target stable blocks written earlier by this task. +// The two immutable inputs may alias each other, but neither aliases the mutable output. // 2. `rayon::scope` synchronises all per-lane tasks before advancing to the next slice — no // read-write race can cross slice boundaries. // 3. Sequential callers (`fill_segment`) hold an exclusive `&mut Matrix` borrow for the duration @@ -1418,10 +1417,10 @@ fn fill_segment( /// Callers must guarantee that, for the indices touched by this call: /// /// - The current segment range `[lane * lane_len + slice * segment_len .. lane * lane_len + (slice -/// + 1) * segment_len]` is exclusively writeable by this task (no other task may read or write -/// within this range while this call runs). -/// - All read indices are stable: blocks at any flat index outside this task's segment range must -/// not be mutated by any other task while this call runs. +/// + 1) * segment_len]` is exclusively writeable by this task. Same-task reads may target +/// already-written positions in that range. +/// - Every immutable input remains stable for the duration of `compress`. The previous and +/// reference inputs may alias each other, but neither may alias the current mutable output. /// /// Both conditions are upheld by: /// - The sequential path's exclusive `&mut Matrix` borrow, OR @@ -1527,14 +1526,13 @@ unsafe fn fill_segment_inner( // SAFETY: // - `prev_idx` is either the previous iteration's `cur_idx` or (on the segment boundary) `lane_base // + lane_len - 1` — both within this task's own lane, never concurrently written. - // - `(ref_lane, ref_index)` per RFC 9106 §3.4 lives in a completed region, never within the current - // slice's still-being-written range. Not concurrently written. + // - `(ref_lane, ref_index)` per RFC 9106 §3.4 is stable for this operation: a cross-lane reference + // excludes the current slice, while a same-lane reference may target only a block already written + // by this task. // - `cur_idx = lane_base + col` is in this task's exclusive segment write range per // `fill_segment_inner`'s safety contract. - // - `prev_idx` and `cur_idx` differ by design (`cur_idx` is the block being written this iteration, - // `prev_idx` is one position earlier or wrapped around to the lane's last block). `ref_idx` is in - // a completed region outside the current segment, never the write target. The three block - // locations are pairwise disjoint. + // - `prev_idx` and `cur_idx` differ by design. `ref_idx` also differs from `cur_idx`; it may equal + // `prev_idx`, which is valid because both are borrowed immutably. // - `compress` was resolved by `active_compress`, so its `required_caps` are a subset of the host's // caps. unsafe { @@ -1570,11 +1568,12 @@ fn fill_slice_sequential( /// Drive one slice's segment fills in parallel: one rayon task per lane. /// -/// Within a slice, every lane writes only to its own segment range and -/// reads only from already-completed regions (RFC 9106 §3.4). The shared -/// [`MatrixView`] therefore carries no aliasing hazard across the -/// `rayon::scope` tasks — see the `MatrixView` and `fill_segment_inner` -/// doc-comments for the full safety argument. +/// Within a slice, every lane writes only to its own segment range. Cross-lane +/// references exclude the current slice; same-lane references may read stable +/// earlier writes from that task (RFC 9106 §3.4). The shared [`MatrixView`] +/// therefore carries no read-write race across the `rayon::scope` tasks — see +/// the `MatrixView` and `fill_segment_inner` doc-comments for the full safety +/// argument. /// /// `rayon::scope` joins all spawned tasks before returning, so the /// `&mut Matrix` borrow is released only after every lane has finished @@ -1611,9 +1610,8 @@ fn fill_slice_parallel( // - Lane `lane`'s write range is `[lane*lane_len + slice*segment_len .. lane*lane_len + // (slice+1)*segment_len]`. This range is pairwise disjoint across lanes, so concurrent tasks // never write the same index. - // - Cross-lane reads (the `(ref_lane, ref_index)` fetch in `fill_segment_inner`) only target blocks - // in completed slices — never the current slice's still-being-written range. Same-lane reads of - // the previous column are within this task's own write range, accessed sequentially. + // - Cross-lane reads (the `(ref_lane, ref_index)` fetch in `fill_segment_inner`) exclude the + // current slice. Same-lane reads may target stable earlier writes from this task. // // No two tasks ever access the same block when at least one is // a writer. The Send/Sync impls on `MatrixView` are sound under @@ -1659,9 +1657,8 @@ fn fill_slice_parallel( /// Dispatch one slice fill: parallel when `parallel` is on and `lanes > 1`, /// and each lane segment has enough work to amortize Rayon scheduling; -/// sequential otherwise. Tiny segments are common in test / benchmark -/// cost shapes (`m=64,p=2` means only 8 blocks per spawned task), where -/// Rayon overhead dominates the actual Argon2 fill on Apple Silicon. +/// sequential otherwise. For example, `m=64,p=2` gives only eight blocks per +/// spawned task, so it remains below the parallel admission threshold. #[inline] fn fill_slice( matrix: &mut Matrix, @@ -1760,7 +1757,13 @@ fn argon2_hash_with_kernel_inner( return Err(Argon2Error::InvalidOutputLen); } - // Compute H0. + // Allocate the matrix before deriving H0 so allocation failure cannot leave + // a password-derived digest in an unwinding stack frame. + let mut matrix = Matrix::new(*params)?; + let lane_len = matrix.lane_len; + let lanes = matrix.lanes; + + // Compute H0 only after all fallible resource acquisition is complete. let mut h0 = { #[cfg(feature = "diag")] { @@ -1776,11 +1779,6 @@ fn argon2_hash_with_kernel_inner( } }; - // Allocate the memory matrix. - let mut matrix = Matrix::new(*params)?; - let lane_len = matrix.lane_len; - let lanes = matrix.lanes; - // Initialise first two blocks per lane. for lane in 0..lanes { let mut buf = [0u8; BLOCK_SIZE]; @@ -1951,9 +1949,8 @@ define_argon2_variant! { /// Argon2i — data-independent indexing variant of Argon2 (RFC 9106). /// /// Memory-access patterns do not depend on the password, closing the - /// cache-timing channel Argon2d accepts. Slower than Argon2d and Argon2id; - /// prefer Argon2id for password hashing unless you specifically need - /// data-independent access patterns throughout. + /// cache-timing channel Argon2d accepts. Prefer Argon2id for password hashing + /// unless you specifically need data-independent access patterns throughout. /// /// # Examples /// diff --git a/src/auth/argon2/power.rs b/src/auth/argon2/power.rs index fcdf8c55..8ff495f0 100644 --- a/src/auth/argon2/power.rs +++ b/src/auth/argon2/power.rs @@ -1,10 +1,9 @@ //! POWER VSX BlaMka compression kernel for Argon2. //! //! Uses `core::simd::u64x2` row pairs and `simd_swizzle!` for cross-pair -//! lane exchange. This stays on the portable-simd surface so the kernel -//! is endian-clean on both POWER8/9/10 little-endian and historical -//! big-endian deployments — VSX intrinsics differ subtly between -//! endianness modes, but the `core::simd` lowering is unambiguous. +//! lane exchange. The portable-simd expression keeps the source independent +//! of VSX intrinsic endianness conventions. Exact compiler lowering remains +//! target- and toolchain-specific generated-code evidence. //! //! # Vectorisation topology //! @@ -20,18 +19,16 @@ //! //! # BlaMka multiply //! -//! `2 · lsb(a) · lsb(b)` lane-wise via masked `u64x2` multiply. POWER's -//! VPMULUDQ-equivalent is `vmulouw` / `vmuleuw` — `core::simd` lowers -//! the masked-multiply pattern to those instructions when it can prove -//! the upper 32 bits are zero, which the explicit `& 0xffffffff` mask -//! makes plain. +//! `2 · lsb(a) · lsb(b)` is expressed lane-wise through masked `u64x2` +//! multiplication. The explicit `& 0xffffffff` mask proves the algorithmic +//! 32-bit operand bound without promising a particular VSX instruction +//! sequence. //! //! # Rotations //! -//! Lane-wise u64 rotate via shift-right + shift-left + OR vector -//! sequence. POWER's `vec_rl` is the native u64 rotate instruction; -//! `core::simd` lowers shift-or pairs to it directly under -//! `target_feature = "vsx"`. +//! Lane-wise u64 rotate is expressed as a shift-right + shift-left + OR +//! vector sequence. Whether the compiler selects a native rotate instruction +//! is a generated-code property, not a source contract. #![cfg(target_arch = "powerpc64")] #![allow(clippy::cast_possible_truncation)] @@ -236,9 +233,7 @@ fn gb(a: &mut Pair, b: &mut Pair, c: &mut Pair, d: &mut Pair) { // ─── Micro-ops ───────────────────────────────────────────────────────────── -/// Lane-wise u64 right-rotate. `core::simd` lowers `(x >> n) | (x << (64-n))` -/// to a single VSX `vrld`-pattern rotate when the shift count is a -/// compile-time constant. +/// Lane-wise u64 right-rotate with a compile-time shift count. #[inline(always)] fn ror(v: u64x2) -> u64x2 { const { assert!(N > 0 && N < 64) } diff --git a/src/auth/argon2/riscv64.rs b/src/auth/argon2/riscv64.rs index a5bf6e90..8e2255e6 100644 --- a/src/auth/argon2/riscv64.rs +++ b/src/auth/argon2/riscv64.rs @@ -1,9 +1,9 @@ //! RISC-V Vector (RVV) BlaMka compression kernel for Argon2. //! -//! 2-u64 scalar pairs that the compiler auto-lowers to RVV at -//! `VL=2 / SEW=64` when the V extension is active. This is the same -//! contract the Blake2b RVV kernel uses; on cores without RVV the -//! sequence still inlines to plain scalar 64-bit ops. +//! Uses two-u64 pairs to expose independent operations to the optimizer while +//! preserving ordinary scalar Rust semantics. Dispatch requires the RISC-V V +//! extension for this backend, but exact RVV lowering is target- and +//! toolchain-specific generated-code evidence. //! //! # Vectorisation topology //! @@ -18,8 +18,7 @@ //! per row (a/b/c/d × {lo, hi}). //! //! Diagonal step: rotate `b` by 1, `c` by 2, `d` by 3 within each -//! 4-lane row. Implemented as plain index swaps; the compiler keeps -//! these in registers under `-O2`. +//! 4-lane row. Implemented as plain index swaps. //! //! # BlaMka multiply //! @@ -29,10 +28,8 @@ //! //! # Rotations //! -//! All four (32, 24, 16, 63) use scalar `u64::rotate_right`. On RVV -//! Zvbb-equipped cores these vectorise to native vector rotates; on -//! older cores they expand to shift+or pairs that the engine -//! pipelines independently per lane. +//! All four (32, 24, 16, 63) use scalar `u64::rotate_right`; instruction +//! selection is left to the target compiler. #![cfg(target_arch = "riscv64")] #![allow(clippy::cast_possible_truncation)] diff --git a/src/auth/asm/rscrypto_rsa_assembly_provenance.tsv b/src/auth/asm/rscrypto_rsa_assembly_provenance.tsv new file mode 100644 index 00000000..68c54ed7 --- /dev/null +++ b/src/auth/asm/rscrypto_rsa_assembly_provenance.tsv @@ -0,0 +1,15 @@ +# rscrypto RSA assembly provenance manifest v1 +schema 1 +transform rsa-aws-lc-sys-0.41.0-transform-v1 +archive aws-lc-sys-0.41.0 1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4 44766fa7daa88e5afc7fc6de3311c48eeeb02f39 +member aws-lc-sys-0.41.0/aws-lc/generated-src/ios-aarch64/crypto/fipsmodule/armv8-mont.S b090cede310cb9911b668fa1a1c8561d3bb1d650b0fb89e62168fd9d8093d1e9 +member aws-lc-sys-0.41.0/aws-lc/generated-src/linux-x86_64/crypto/fipsmodule/x86_64-mont.S 97164bd82dbbd52b5a2b2cd839223b942d703209fad63174a4c75505d693e2cc +member aws-lc-sys-0.41.0/aws-lc/generated-src/linux-x86_64/crypto/fipsmodule/x86_64-mont5.S 0d3d1e4dc0db6822d087c1e2b91a68056eaf7d07d63725f65c45744b7c3e7d11 +member aws-lc-sys-0.41.0/aws-lc/crypto/fipsmodule/bn/asm/armv8-mont.pl f1b84fceebcfd32d1f5e106ad3123204c7f3b0d0b226265d87dd819e43ec46b8 +member aws-lc-sys-0.41.0/aws-lc/crypto/fipsmodule/bn/asm/x86_64-mont.pl 0a010a6176e9568358d816ea0acd62e4f9a1a88763a5963f638b56f55230044e +member aws-lc-sys-0.41.0/aws-lc/crypto/fipsmodule/bn/asm/x86_64-mont5.pl 2e02dc6d7e8727502db6c61bbd6d554ae13621b630364f954ce6c23e5333d63b +member aws-lc-sys-0.41.0/aws-lc/crypto/perlasm/arm-xlate.pl cf58ab9cf9c02c849aca10a83c0e640178bf6ff0bb98a34de54c56d5c2856f80 +member aws-lc-sys-0.41.0/aws-lc/crypto/perlasm/x86_64-xlate.pl 2745d9400c9a5d2385da57eceb94b4c1ba8991cfe4ee90ba017fba7b361fe5b7 +output src/auth/asm/rscrypto_rsa_bignum_mont_apple.s 1439 32065 3e723bd775c6e216d9525d470d6d082a83e989f7988db0355c0ad9cd5bfa5072 +output src/auth/asm/rscrypto_rsa_bignum_mont_aarch64_elf.s 1439 32095 f28fa8f4f02e0427288c717fc5666b7c07f0f8fc0c7ea5bb21721b1ffe3dc4dc +output src/auth/asm/rscrypto_rsa_x86_64_elf.S 2530 42855 1b59e82724d10b06decbc8a9fbc88f9e04cefdee0524ef752c37ccf422f729a3 diff --git a/src/auth/asm/rscrypto_signature_assembly_provenance.tsv b/src/auth/asm/rscrypto_signature_assembly_provenance.tsv new file mode 100644 index 00000000..0bfa7027 --- /dev/null +++ b/src/auth/asm/rscrypto_signature_assembly_provenance.tsv @@ -0,0 +1,39 @@ +# Non-RSA signature assembly provenance. Source members are ordered. +schema 1 +upstream https://github.com/awslabs/s2n-bignum.git +output src/auth/asm/rscrypto_bignum_mod_n256_aarch64_apple_darwin.s d498257b656e9a9cf7ac39514f34f0ebf6e8b51b444f248b599c8e21f8c4e09e 02d36faf23372d5ab50924c7e0685bcf809ec6430f2566c3585fe588a1aa4428 471fca76a9079753aab938ba35ef55ec22717d89 A64_MACHO_ECDSA_V1 arm/p256/bignum_mod_n256.S=34053764b412198ec659a312c2ee0b93705c3ab15451914e01e809241f88358d +output src/auth/asm/rscrypto_bignum_mod_n256_aarch64_unknown_linux.s d6bc970e8ab6990d018ab871200985fbe598ca0a90439fd195ec2c1ca5243d43 c27e85f962bcff47dd97a21bd8007e7872b9bbbd3b7ec06508271ac9b41f8a9d 471fca76a9079753aab938ba35ef55ec22717d89 A64_ELF_ECDSA_SPLIT_V1 arm/p256/bignum_mod_n256.S=34053764b412198ec659a312c2ee0b93705c3ab15451914e01e809241f88358d +output src/auth/asm/rscrypto_bignum_mod_n256_x86_64_unknown_linux.S 87e0d2fd00384c9373e943ada52150c882f518009bceb3a18b656d823dadedf1 3bf741992aecf01c25203f346c57e6ff054915884ec6c6a5523d9d19898a1d7f 471fca76a9079753aab938ba35ef55ec22717d89 X86_ELF_ECDSA_GLOBAL_V1 x86_att/p256/bignum_mod_n256.S=1814e7d60f5d79c292f044d75e2d7d03024fc2afe10f229d1991deb52519b2a4 +output src/auth/asm/rscrypto_bignum_mod_n384_aarch64_apple_darwin.s f62fe966377a6b3855eddde911839fe2c22c3e8c7b3a9ade935af106efcaa07e b72f04778b1f6aad57fe1ea74e6dd5bdceddfdac6df9a53faf1d2e4b73d02f09 471fca76a9079753aab938ba35ef55ec22717d89 A64_MACHO_ECDSA_V1 arm/p384/bignum_mod_n384.S=c8f4892a589502aae825fa8b3443089ad90faac2b8874e9348f4fc94bf4a7ae1 +output src/auth/asm/rscrypto_bignum_mod_n384_aarch64_unknown_linux.s 4f779028acf389713bda923bda71e427b11f0de44ad75dd2bad9f671bf4ce3e6 64cdf6d7e4f94c3c379e2f6e4dc920aca6c6f14077f6345de0ad9dd20f831feb 471fca76a9079753aab938ba35ef55ec22717d89 A64_ELF_ECDSA_SPLIT_V1 arm/p384/bignum_mod_n384.S=c8f4892a589502aae825fa8b3443089ad90faac2b8874e9348f4fc94bf4a7ae1 +output src/auth/asm/rscrypto_bignum_modinv_aarch64_apple_darwin.s 510f5d4284df8be3b9168235665840037a43dc2018fe775d70376e0def116186 a1c90bc39e737232e8266385be6e6d65097b3d58e92a404fc704f7673261a431 471fca76a9079753aab938ba35ef55ec22717d89 A64_MACHO_ECDSA_V1 arm/generic/bignum_modinv.S=6ab952bea2fa310092c45f96720e45c9523f1c3baccb2925f9e29ca84a9e6b42 +output src/auth/asm/rscrypto_bignum_modinv_aarch64_unknown_linux.s 02b7ba80d15af8f74e6d77eeaa250c450b02ec48ca958fd1887b46e11d1e2d55 f54e120122cb40db6048d31d92380ef19dfd6b7d214cb8b05336350b6a981c99 471fca76a9079753aab938ba35ef55ec22717d89 A64_ELF_ECDSA_SPLIT_V1 arm/generic/bignum_modinv.S=6ab952bea2fa310092c45f96720e45c9523f1c3baccb2925f9e29ca84a9e6b42 +output src/auth/asm/rscrypto_bignum_modinv_x86_64_unknown_linux.S 076dfaa54ba79ae2b87bdf26f4125baca169375e5b7b726c888f38b26aa78a5c 50f573a0875a0bc696c1e4e0644bd41fa7037e99852cc171fb58e9729722fe57 471fca76a9079753aab938ba35ef55ec22717d89 X86_ELF_ECDSA_GLOBAL_V1 x86_att/generic/bignum_modinv.S=060e4b1f9852ebcbdcadefb6a804442861548b044bab02cd8d98f6f28475dc3f +output src/auth/asm/rscrypto_bignum_montinv_p384_aarch64_apple_darwin.s 73244e2de1a1277f0c78803875c81d24a08b4d36bf83dd612a688e0021640f83 5af87fb86148a326c147b65bf5701596062d7c4c1e765d27aff2124b26b19f54 471fca76a9079753aab938ba35ef55ec22717d89 A64_MACHO_ECDSA_V1 arm/p384/bignum_montinv_p384.S=ad1b6d4c921d284e2b4819e4b83ee451571487fa52a558926b3ba9ce7a352a41 +output src/auth/asm/rscrypto_bignum_montinv_p384_aarch64_unknown_linux.s ad13f0761399295d36f245da4fdca82cb150c38366ff934821b1e445dd993647 97e992049060cfa2dcf68194d9c0f83d17201189f6f36162af09e12cc10041b8 471fca76a9079753aab938ba35ef55ec22717d89 A64_ELF_ECDSA_SPLIT_V1 arm/p384/bignum_montinv_p384.S=ad1b6d4c921d284e2b4819e4b83ee451571487fa52a558926b3ba9ce7a352a41 +output src/auth/asm/rscrypto_bignum_montinv_p384_x86_64_unknown_linux.S b09e951bd5597c9ad676962aa3ce10eb57605fb9e45c994468363d606043b638 007ce40331ada1663bb1c1df52053c82fe7904f9c3278ea3f82b9a4841cba426 471fca76a9079753aab938ba35ef55ec22717d89 X86_ELF_ECDSA_GLOBAL_V1 x86_att/p384/bignum_montinv_p384.S=2925d2063b0b5944ba16d779d7e10873873025a93eec02c0f4dd51e9beb63cc9 +output src/auth/asm/rscrypto_bignum_montmul_p384_aarch64_apple_darwin.s b373c213342b64a52903a697a4dd56ebaa3e76d5965b2379a721c584a3f6b295 db3c062e2431a8f0d25a98dfa89945f271ae218c8e1d9f76912e8b8486edf6d5 471fca76a9079753aab938ba35ef55ec22717d89 A64_MACHO_ECDSA_V1 arm/p384/bignum_montmul_p384.S=bbbe66f0f71abf979086591a9463d05f48ffc4d0840c2a2ad4268dadf18cd12b +output src/auth/asm/rscrypto_bignum_montmul_p384_aarch64_unknown_linux.s 37ab8b9040283f916cb95e3497ab3c72c00117aa077250f96dc654d4bc35732f 625c4a971f7ad010b27caa40c40ccaf2eab7372cfc9f9b3b992d9b4d6e280993 471fca76a9079753aab938ba35ef55ec22717d89 A64_ELF_ECDSA_SPLIT_V1 arm/p384/bignum_montmul_p384.S=bbbe66f0f71abf979086591a9463d05f48ffc4d0840c2a2ad4268dadf18cd12b +output src/auth/asm/rscrypto_bignum_montmul_p384_alt_x86_64_unknown_linux.S e7f94bdc3e6051bf4b48aa39ddfb8038a81cde4867cda6da056cd2006ee48af1 7f51726f3398b72dbdb76a60466b8e737c7d4098a667df397d398bba409ccb74 471fca76a9079753aab938ba35ef55ec22717d89 X86_ELF_ECDSA_GLOBAL_V1 x86_att/p384/bignum_montmul_p384_alt.S=5ec8709b69b0b31324d724d2964a0114848ced496f7f8199ffff461b78c233b4 +output src/auth/asm/rscrypto_bignum_montmul_p384_x86_64_unknown_linux.S 593ac3fc5dcd6bc5fe9e444fa792e91fbbc83a392ff21f52b19d074c45837b7f e92bb76afc71559c755fc2b85f166de1501cae91c8bccbaa15f5194f5f4effd0 471fca76a9079753aab938ba35ef55ec22717d89 X86_ELF_ECDSA_GLOBAL_V1 x86_att/p384/bignum_montmul_p384.S=d9521c0e0c8308a3b7426a6ebc84aa987e79dbaca075876f083565adcf425b4f +output src/auth/asm/rscrypto_bignum_montsqr_p384_aarch64_apple_darwin.s b5849beb31f01fbd9a50205643398ad8b0c3d45c62520c0b19f2aae18a37cadc f8e933b4088cd6b06ae85c0c54d0fc5957d52ac97e1c032f311533d00ad23030 471fca76a9079753aab938ba35ef55ec22717d89 A64_MACHO_ECDSA_V1 arm/p384/bignum_montsqr_p384.S=e151526b29b9a284ecbb374ad579f3aa9f9a9b8f64868e88d9f05f78e99367cc +output src/auth/asm/rscrypto_bignum_montsqr_p384_aarch64_unknown_linux.s 638091ee285dd42c34275bccbcea92b25e13bb512089d7ac0866836a0f3a7cbc 0fdbb0b1341d5922440688bd01e7cd33bde0305fbc5408bce5a49b14eefbfa7a 471fca76a9079753aab938ba35ef55ec22717d89 A64_ELF_ECDSA_SPLIT_V1 arm/p384/bignum_montsqr_p384.S=e151526b29b9a284ecbb374ad579f3aa9f9a9b8f64868e88d9f05f78e99367cc +output src/auth/asm/rscrypto_bignum_montsqr_p384_alt_x86_64_unknown_linux.S a6e91e141ce6788fc483288f36a4761f3188a1ca2363dcf2154e88f141ebf677 12920fe9fc8a2648e496f8c5815372547b221c8177792a81854933b3d0cda280 471fca76a9079753aab938ba35ef55ec22717d89 X86_ELF_ECDSA_GLOBAL_V1 x86_att/p384/bignum_montsqr_p384_alt.S=3a8ede9859441c0613d0b569ec3bd4aaf453f49149d6222dcef711bb00f557d0 +output src/auth/asm/rscrypto_bignum_montsqr_p384_x86_64_unknown_linux.S 8f7480af7c61413765edc2b2ffc8e005ff01d6e2e7a605650aa4036a6d717bdd f2c75b84fa58f5d25f9d51014d647f859b5cf6d39e9029175479ca8d927833c3 471fca76a9079753aab938ba35ef55ec22717d89 X86_ELF_ECDSA_GLOBAL_V1 x86_att/p384/bignum_montsqr_p384.S=5e2b95a1592b11aed7d3a14d8d02f5b8a0669614b7fe7e8ac5bc89959918b01b +output src/auth/asm/rscrypto_p256_scalarmulbase_alt_aarch64_apple_darwin.s 8c42bbdf05608f9b44090973f418815b9a5466b1b364dff008e1ea7b3af71354 69082c321fd54adc790dba913cd467b7feace5593cf74fc1619f23aedef27de1 471fca76a9079753aab938ba35ef55ec22717d89 A64_MACHO_ECDSA_V1 arm/p256/p256_scalarmulbase_alt.S=d3fd34fc1dde9fae9688903b76964fb26f0764a3ccff60e1c81b55d4181b3bb3 +output src/auth/asm/rscrypto_p256_scalarmulbase_alt_aarch64_unknown_linux.s dd6e2821c9dcd1371bf4b235e7562e97c24d1b4b653bdafac065fc9da3cf16ae ac318a2df0b2d194d1752685ae0ab9aaad86dabcd4d3288ddf3336b0916ec22f 471fca76a9079753aab938ba35ef55ec22717d89 A64_ELF_ECDSA_SPLIT_V1 arm/p256/p256_scalarmulbase_alt.S=d3fd34fc1dde9fae9688903b76964fb26f0764a3ccff60e1c81b55d4181b3bb3 +output src/auth/asm/rscrypto_p256_scalarmulbase_alt_x86_64_unknown_linux.S 60e2fe6527256d6242e22020663c765af608cb1be99725a488618e405f4d023c 7d629d440cedf21e7efad09a8fe3da56d4c7c15693d64c95d01139c9dbc63623 471fca76a9079753aab938ba35ef55ec22717d89 X86_ELF_ECDSA_GLOBAL_V1 x86_att/p256/p256_scalarmulbase_alt.S=62e47d1178004447dbdcffdc7756ccabb391a494763d5b4fd8606169f919e11c +output src/auth/asm/rscrypto_p256_scalarmulbase_x86_64_unknown_linux.S 619acce477eb16b78ebf555f333b4ad0cbcf6c0260346c09f5380e731abf45bf 6eb8dfc6ab19fbae644a30efac59f01ab169585c92204a122b5fd928ecf1051b 471fca76a9079753aab938ba35ef55ec22717d89 X86_ELF_ECDSA_GLOBAL_V1 x86_att/p256/p256_scalarmulbase.S=c76fcbf6e8a0646482f22da579d96a23528c571a2c280daded481c4e3fe45382 +output src/auth/asm/rscrypto_p384_montjdouble_alt_aarch64_apple_darwin.s 44304c7d3cd2fad34b317da6edf5741903a4fe791334b2a7e2e003fc15030cd4 d766877a454d6aee84825c26098d4339b6dfb0d17aa7db8051e2b752a2aacfff 471fca76a9079753aab938ba35ef55ec22717d89 A64_MACHO_ECDSA_V1 arm/p384/p384_montjdouble_alt.S=2c6a2405d21cee6227dd930281be6634dbdf64bf437f66e598c6771700f929f2 +output src/auth/asm/rscrypto_p384_montjdouble_alt_aarch64_unknown_linux.s f7b2f87b4f541f2f500616315d3fe521d49d6586aa0ae691c515c3471397d401 0e287156833cb159e35c38f436561088fc90f842fd7dd10723f3cd41f9f4c13b 471fca76a9079753aab938ba35ef55ec22717d89 A64_ELF_ECDSA_SPLIT_V1 arm/p384/p384_montjdouble_alt.S=2c6a2405d21cee6227dd930281be6634dbdf64bf437f66e598c6771700f929f2 +output src/auth/asm/rscrypto_p384_montjmixadd_alt_aarch64_apple_darwin.s 0821b8fe2bd578127e33dd8a2d618e023fcd7c8f6b5534813048bb188880e564 2606dfbe4d1b8dd1e0ad942d0579d55eed5440a975d938c12a829887b44de046 471fca76a9079753aab938ba35ef55ec22717d89 A64_MACHO_ECDSA_V1 arm/p384/p384_montjmixadd_alt.S=fa0f46c080a4da771553024db38512f17b741fbb6b8d4de0d77d56d230523395 +output src/auth/asm/rscrypto_p384_montjmixadd_alt_aarch64_unknown_linux.s 1224d7c409a8fb23c577a95e4edd7cc29ede93f5cdbd01d31d440cbb3ec56e21 35f03c2efc5fb8d61d18121cb8d173bc41d5e1a52d2f5011443c1243d7b6ef46 471fca76a9079753aab938ba35ef55ec22717d89 A64_ELF_ECDSA_SPLIT_V1 arm/p384/p384_montjmixadd_alt.S=fa0f46c080a4da771553024db38512f17b741fbb6b8d4de0d77d56d230523395 +output src/auth/ed25519/asm/rscrypto_ed25519_aarch64_apple_darwin.s fc52878b016ce2b42494538fef4519c0b69752d761e338814ec607b0a0e43ae2 27bb7659679f46c077659154d932e4ae5db539b5fb5d1bacbe925eb8e753ea4d 471fca76a9079753aab938ba35ef55ec22717d89 A64_MACHO_CONCAT_TOKEN_V1 arm/curve25519/edwards25519_decode_alt.S=a890081773f5a48194f6891b4a7555c7c6289d528c90b5057d225b783051c35f,arm/curve25519/edwards25519_scalarmuldouble_alt.S=d18ce8e5171301bb8693637ca829a143653450092f70509d12d7b7240f789db5 +output src/auth/ed25519/asm/rscrypto_ed25519_aarch64_unknown_linux.s 609d6a5c6ca136a2edab807b678b3d332d726cb3b6383f297ead2872671ccac4 e6dcd4a4f519cf2db169e895ccedd5d63253134a19120f65010f09db9e6ceb08 471fca76a9079753aab938ba35ef55ec22717d89 A64_ELF_ED25519_GLOBAL_V1 arm/curve25519/edwards25519_decode_alt.S=a890081773f5a48194f6891b4a7555c7c6289d528c90b5057d225b783051c35f,arm/curve25519/edwards25519_scalarmuldouble_alt.S=d18ce8e5171301bb8693637ca829a143653450092f70509d12d7b7240f789db5 +output src/auth/ed25519/asm/rscrypto_ed25519_scalarmulbase_aarch64_apple_darwin.s f4178d7aa176bd18feea0fee886f4574aab5df462cb523afbccb2530ec855f8d ba26a70e7400c699c3cab909b94bfb12cbf2e68893a8622a78960b840ff4a775 471fca76a9079753aab938ba35ef55ec22717d89 A64_MACHO_CONCAT_TOKEN_V1 arm/curve25519/edwards25519_scalarmulbase_alt.S=b55827ca7f0e9a2d977512db8d4ecbcb0499f7294e068a1f4b1131bc96e50e2a +output src/auth/ed25519/asm/rscrypto_ed25519_scalarmulbase_aarch64_unknown_linux.s 6b4166fb859e85ac9282c00db1d116c979c232719481d03be443b28621f74d22 9ed2844005786601c988a6cc6b1c62b42fd0d7a5237bfdd34347cfd93b512295 471fca76a9079753aab938ba35ef55ec22717d89 A64_ELF_ED25519_GLOBAL_V1 arm/curve25519/edwards25519_scalarmulbase_alt.S=b55827ca7f0e9a2d977512db8d4ecbcb0499f7294e068a1f4b1131bc96e50e2a +output src/auth/ed25519/asm/rscrypto_ed25519_scalarmulbase_alt_x86_64_unknown_linux.s 5b4dfac7e7d4a2d2d1f499959ed310c1a17ad6c33e2fbcc3602f2d79d6cd5b13 cf781f21c4f78078b6de8c00751c9697e74c3d2021b0c4d84001c63765754802 c19516a30de81f9e664dccdfc79dbf8fb109276d X86_ELF_ED25519_GLOBAL_V1 x86_att/curve25519/edwards25519_scalarmulbase_alt.S=d397dec9f4d6baf99de7d9b7804c8a81dd549edd6e76aa6a86faa4786536fa33 +output src/auth/ed25519/asm/rscrypto_ed25519_scalarmulbase_x86_64_unknown_linux.s 113ed6c2de25ef6079cac6dfa44ac8e7a86ef702e810101200e7c88189e84794 460894c90820cc9ec498cc0de1404c9cfb78e0da2c815914b7d79caad780ea2c c19516a30de81f9e664dccdfc79dbf8fb109276d X86_ELF_ED25519_GLOBAL_V1 x86_att/curve25519/edwards25519_scalarmulbase.S=3d741fd1dca16389504662404851f785f1822912cd1cf177050a3534d7aace0b +output src/auth/x25519/asm/rscrypto_x25519_aarch64_apple_darwin.s f955851b0119740929411c2eccd4d2b84196ef2831740b85f4a9e3978ccda094 e811d4473b9ebaf699c95aee1e948c6f667adbc5eaa5c64289599d2f64801397 471fca76a9079753aab938ba35ef55ec22717d89 A64_MACHO_CONCAT_TOKEN_V1 arm/curve25519/curve25519_x25519_byte_alt.S=c44b7a1af90de5c413ef707d67da439e995ec23fd5801f7bf20956b2ec9e3339,arm/curve25519/curve25519_x25519base_byte_alt.S=f8c555ba9989f4cf2c5376db26b48c302ee0df695d2960a83933e96dbcfa1db7 +output src/auth/x25519/asm/rscrypto_x25519_aarch64_unknown_linux.s 5365eac66498023820ef2c6ba5d7bc3b1ca01f5087e3dc73fd92070deb752ac8 e607317c91595ec553fcc80ec4a14482874fa487fd134eab2a6daab377cceb98 471fca76a9079753aab938ba35ef55ec22717d89 A64_ELF_X25519_TOKEN_V1 arm/curve25519/curve25519_x25519_byte_alt.S=c44b7a1af90de5c413ef707d67da439e995ec23fd5801f7bf20956b2ec9e3339,arm/curve25519/curve25519_x25519base_byte_alt.S=f8c555ba9989f4cf2c5376db26b48c302ee0df695d2960a83933e96dbcfa1db7 +output src/auth/x25519/asm/rscrypto_x25519_x86_64_linux.s b366027331f8edd7288e30338d399ecd9eb11562e989a95f91797a79f52bd9be 305afda39492e659c33e9c138a3ceb0b1121c24f7ddffe804bbd897d793781e0 333cdfcd91a62d15954ecca1124544b8587f86de X86_ELF_X25519_CONCAT_TOKEN_V1 x86_att/curve25519/curve25519_x25519.S=bf1425f906390550209705e52c7c3625cbf411c0d6b8075d5f361194fbff2800,x86_att/curve25519/curve25519_x25519base.S=8d23515f9f5185bae1d2f0c4689ec9ad17366f4d197b025ad567293f85bd636a,x86_att/curve25519/curve25519_x25519_alt.S=bc5d4ad630baed401b3e085694c20a7a3498b44c88493ac2368491ba4a34afb8,x86_att/curve25519/curve25519_x25519base_alt.S=8d3f557d4ab0a8b5211b79cd38fabe22a96a95c1efd484bbb74dd4fae7af67a6 diff --git a/src/auth/ecdsa.rs b/src/auth/ecdsa.rs index feb03e07..5ebffd1d 100644 --- a/src/auth/ecdsa.rs +++ b/src/auth/ecdsa.rs @@ -3402,6 +3402,28 @@ pub fn diag_ecdsa_p256_select_signing_generator_affine_limb_digest(digit: u8) -> out } +#[cfg(all( + feature = "diag", + feature = "ecdsa-p256", + any( + all(target_arch = "aarch64", any(target_os = "macos", target_os = "linux")), + all(target_arch = "x86_64", target_os = "linux") + ) +))] +#[doc(hidden)] +#[unsafe(no_mangle)] +#[inline(never)] +pub(crate) fn diag_zeroize_ecdsa_p256_platform_scratch(wide: [u8; 64]) -> u64 { + let wide = ZeroizingBytes::new(wide); + let reduced = SecretScalar::new(reduce_wide_order_nonzero(wide.as_array(), &P256_ORDER_MODULUS)); + let inverse = SecretScalar::new( + reduced + .value() + .inv_mod_ct_montgomery(&P256_ORDER_MODULUS, P256_ORDER_MINUS_TWO), + ); + core::hint::black_box(inverse.value().0[0]) +} + #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_nonce_reduce_limb_digest(secret: [u8; 32], message: &[u8]) -> [u64; 4] { let secret = ZeroizingBytes::new(secret); @@ -3523,6 +3545,26 @@ pub fn diag_ecdsa_p384_select_signing_generator_affine_limb_digest(digit: u8) -> out } +#[cfg(all( + feature = "diag", + feature = "ecdsa-p384", + target_arch = "aarch64", + any(target_os = "macos", target_os = "linux") +))] +#[doc(hidden)] +#[unsafe(no_mangle)] +#[inline(never)] +pub(crate) fn diag_zeroize_ecdsa_p384_platform_scratch(wide: [u8; 96]) -> u64 { + let wide = ZeroizingBytes::new(wide); + let reduced = SecretScalar::new(reduce_wide_order_nonzero(wide.as_array(), &P384_ORDER_MODULUS)); + let inverse = SecretScalar::new( + reduced + .value() + .inv_mod_ct_montgomery(&P384_ORDER_MODULUS, P384_ORDER_MINUS_TWO), + ); + core::hint::black_box(inverse.value().0[0]) +} + #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_nonce_reduce_limb_digest(secret: [u8; 48], message: &[u8]) -> [u64; 6] { let secret = ZeroizingBytes::new(secret); diff --git a/src/auth/ecdsa_aarch64_asm.rs b/src/auth/ecdsa_aarch64_asm.rs index c4fe9838..5d515d6e 100644 --- a/src/auth/ecdsa_aarch64_asm.rs +++ b/src/auth/ecdsa_aarch64_asm.rs @@ -9,6 +9,8 @@ use core::arch::global_asm; +use super::ZeroizingWords; + #[path = "ecdsa_aarch64_tables.rs"] mod ecdsa_aarch64_tables; @@ -92,7 +94,7 @@ pub(super) fn p256_scalarmulbase_generator(scalar: &[u64; 4]) -> [u64; 8] { #[inline] pub(super) fn p256_reduce_order_64(bytes: &[u8; 64]) -> [u64; 4] { - let input = words_from_be_bytes_reversed::<8, 64>(bytes); + let input = ZeroizingWords::new(words_from_be_bytes_reversed::<8, 64>(bytes)); let mut out = [0u64; 4]; // SAFETY: P-256 order reduction call because: // 1. This module is compiled only for supported AArch64, matching the embedded assembly ABI. @@ -100,13 +102,13 @@ pub(super) fn p256_reduce_order_64(bytes: &[u8; 64]) -> [u64; 4] { // 3. `input` has eight `u64` limbs and `len` is 8, so the assembly reads exactly the provided // input. // 4. The routine runs a fixed-size reduction for public length 8; the reduced value may be secret. - unsafe { rscrypto_bignum_mod_n256(out.as_mut_ptr(), 8, input.as_ptr()) }; + unsafe { rscrypto_bignum_mod_n256(out.as_mut_ptr(), 8, input.as_array().as_ptr()) }; out } #[inline] pub(super) fn p384_reduce_order_96(bytes: &[u8; 96]) -> [u64; 6] { - let input = words_from_be_bytes_reversed::<12, 96>(bytes); + let input = ZeroizingWords::new(words_from_be_bytes_reversed::<12, 96>(bytes)); let mut out = [0u64; 6]; // SAFETY: P-384 order reduction call because: // 1. This module is compiled only for supported AArch64, matching the embedded assembly ABI. @@ -114,7 +116,7 @@ pub(super) fn p384_reduce_order_96(bytes: &[u8; 96]) -> [u64; 6] { // 3. `input` has twelve `u64` limbs and `len` is 12, so the assembly reads exactly the provided // input. // 4. The routine runs a fixed-size reduction for public length 12; the reduced value may be secret. - unsafe { rscrypto_bignum_mod_n384(out.as_mut_ptr(), 12, input.as_ptr()) }; + unsafe { rscrypto_bignum_mod_n384(out.as_mut_ptr(), 12, input.as_array().as_ptr()) }; out } @@ -159,7 +161,7 @@ pub(super) fn p384_field_inverse(value: &[u64; 6]) -> [u64; 6] { #[inline] pub(super) fn scalar_inverse(value: &[u64; L], modulus: &[u64; L]) -> [u64; L] { let mut out = [0u64; L]; - let mut tmp = [0u64; 18]; + let mut tmp = ZeroizingWords::<18>::zeroed(); debug_assert!(L == 4 || L == 6); // SAFETY: scalar-order inverse call because: // 1. This module is compiled only for supported AArch64, matching the embedded assembly ABI. @@ -173,7 +175,7 @@ pub(super) fn scalar_inverse(value: &[u64; L], modulus: &[u64; L out.as_mut_ptr(), value.as_ptr(), modulus.as_ptr(), - tmp.as_mut_ptr(), + tmp.as_mut_array().as_mut_ptr(), ) }; out diff --git a/src/auth/ecdsa_x86_64_asm.rs b/src/auth/ecdsa_x86_64_asm.rs index 8e17bdc6..79e0dcbe 100644 --- a/src/auth/ecdsa_x86_64_asm.rs +++ b/src/auth/ecdsa_x86_64_asm.rs @@ -9,6 +9,7 @@ use core::arch::global_asm; +use super::ZeroizingWords; use crate::platform::{self, caps::x86}; #[path = "ecdsa_aarch64_tables.rs"] @@ -112,7 +113,7 @@ pub(super) fn p256_scalarmulbase_generator(scalar: &[u64; 4]) -> [u64; 8] { #[inline] pub(super) fn p256_reduce_order_64(bytes: &[u8; 64]) -> [u64; 4] { - let input = words_from_be_bytes_reversed::<8, 64>(bytes); + let input = ZeroizingWords::new(words_from_be_bytes_reversed::<8, 64>(bytes)); let mut out = [0u64; 4]; // SAFETY: P-256 order reduction call because: // 1. This module is compiled only for Linux x86-64 System V, matching the embedded assembly ABI. @@ -120,7 +121,7 @@ pub(super) fn p256_reduce_order_64(bytes: &[u8; 64]) -> [u64; 4] { // 3. `input` has eight `u64` limbs and `len` is 8, so the assembly reads exactly the provided // input. // 4. The routine runs a fixed-size reduction for public length 8; the reduced value may be secret. - unsafe { rscrypto_bignum_mod_n256(out.as_mut_ptr(), 8, input.as_ptr()) }; + unsafe { rscrypto_bignum_mod_n256(out.as_mut_ptr(), 8, input.as_array().as_ptr()) }; out } @@ -193,7 +194,7 @@ pub(super) fn p384_field_inverse(value: &[u64; 6]) -> [u64; 6] { #[inline] pub(super) fn scalar_inverse(value: &[u64; L], modulus: &[u64; L]) -> [u64; L] { let mut out = [0u64; L]; - let mut tmp = [0u64; 18]; + let mut tmp = ZeroizingWords::<18>::zeroed(); debug_assert!(L == 4 || L == 6); // SAFETY: scalar-order inverse call because: // 1. This module is compiled only for Linux x86-64 System V, matching the embedded assembly ABI. @@ -207,7 +208,7 @@ pub(super) fn scalar_inverse(value: &[u64; L], modulus: &[u64; L out.as_mut_ptr(), value.as_ptr(), modulus.as_ptr(), - tmp.as_mut_ptr(), + tmp.as_mut_array().as_mut_ptr(), ) }; out diff --git a/src/auth/ed25519.rs b/src/auth/ed25519.rs index 7d48359d..2733617c 100644 --- a/src/auth/ed25519.rs +++ b/src/auth/ed25519.rs @@ -174,8 +174,9 @@ impl Ed25519SecretKey { /// Sign a message with this secret key. #[must_use] pub fn sign(&self, message: &[u8]) -> Ed25519Signature { - let public = self.public_key(); - sign_with_secret(self, &public, message) + let expanded = hash::ExpandedSecret::from_secret_key(self); + let public = public_key_from_scalar(expanded.scalar_bytes()); + sign_with_expanded(&expanded, &public, message) } } @@ -572,12 +573,6 @@ pub fn verify( } } -#[must_use] -fn sign_with_secret(secret: &Ed25519SecretKey, public: &Ed25519PublicKey, message: &[u8]) -> Ed25519Signature { - let expanded = hash::ExpandedSecret::from_secret_key(secret); - sign_with_expanded(&expanded, public, message) -} - #[must_use] fn sign_with_expanded(expanded: &hash::ExpandedSecret, public: &Ed25519PublicKey, message: &[u8]) -> Ed25519Signature { let mut secret_scalar = expanded.scalar_words(); @@ -896,8 +891,7 @@ fn public_key_from_scalar(scalar_bytes: &[u8; SECRET_KEY_LENGTH]) -> Ed25519Publ /// available path. /// /// Uses wNAF-based Straus on IFMA platforms: wNAF(8) for the basepoint -/// scalar (~28 additions) + wNAF(5) for the public-key scalar (~43 -/// additions) = ~71 total, vs ~128 with the old radix-16 approach. +/// scalar and wNAF(5) for the public-key scalar. /// /// AVX2 wNAF Straus remains as the fallback for pre-IFMA hardware. #[must_use] @@ -908,7 +902,7 @@ fn straus_dispatch(s: &[u8; 32], h: &[u8; 32], a: &point::ExtendedPoint) -> poin let caps = crate::platform::caps(); - // IFMA + wNAF path: ~45% fewer additions than radix-16. + // IFMA + wNAF path. if caps.has(x86::AVX512IFMA) && caps.has(x86::AVX512VL) && caps.has(x86::AVX2) { // SAFETY: AVX-512 IFMA + VL + AVX2 confirmed by runtime detection. return unsafe { point_avx2::straus_wnaf_vartime_ifma(s, h, a) }; @@ -1034,10 +1028,11 @@ mod tests { #[test] fn keypair_signs_and_public_key_verifies() { let secret = Ed25519SecretKey::from_bytes([0x55; Ed25519SecretKey::LENGTH]); - let keypair = Ed25519Keypair::from_secret_key(secret); + let keypair = Ed25519Keypair::from_secret_key(secret.duplicate_secret()); let message = b"rscrypto-ed25519"; let signature = keypair.sign(message); + assert_eq!(secret.sign(message), signature); assert!(keypair.public_key().verify(message, &signature).is_ok()); assert!(verify(message, &keypair.public_key(), &signature).is_ok()); } diff --git a/src/auth/ed25519/field_ifma.rs b/src/auth/ed25519/field_ifma.rs index 1eddc5be..d27c7730 100644 --- a/src/auth/ed25519/field_ifma.rs +++ b/src/auth/ed25519/field_ifma.rs @@ -546,8 +546,8 @@ impl FieldElement51x4 { /// This is the key optimization for IFMA point operations: `diff_sum()` /// produces limbs up to 53 bits (51-bit reduced + 2-bit bias from add/sub), /// but IFMA's `vpmadd52luq` truncates inputs to 52 bits. The standard - /// `mul()` therefore requires a `reduce()` call (17 insns, ~18 cycle - /// latency) before each multiply after `diff_sum()`. + /// `mul()` therefore requires a `reduce()` call before each multiply after + /// `diff_sum()`. /// /// `mul_unreduced()` handles the 53rd bit by decomposing each operand /// into a 52-bit low part (suitable for IFMA) and a 0/1 high bit, then @@ -1217,8 +1217,8 @@ impl FieldElement51x4 { /// /// The Hamburg scaling constants `(d2, d2, 2·d2, 2·d1)` fit in 18 bits, /// so their radix-51 representation has non-zero content only in limb 0. - /// This needs only 10 IFMA ops (5 lo + 5 hi) instead of the full 50-op - /// schoolbook, saving ~40 IFMA ops per constant multiply. + /// This uses 5 low and 5 high IFMA operations instead of the full + /// schoolbook multiplication. /// /// # Precondition /// diff --git a/src/auth/ed25519/hash.rs b/src/auth/ed25519/hash.rs index f4e1776c..0e322a5a 100644 --- a/src/auth/ed25519/hash.rs +++ b/src/auth/ed25519/hash.rs @@ -16,7 +16,7 @@ use super::{ use crate::{hashes::crypto::Sha512, traits::ct}; /// Expanded secret-key material derived from SHA-512(secret_key). -#[derive(Clone, PartialEq, Eq)] +#[derive(Clone)] pub(crate) struct ExpandedSecret { scalar_bytes: [u8; SECRET_KEY_LENGTH], nonce_prefix: [u8; SECRET_KEY_LENGTH], diff --git a/src/auth/ed25519/point_avx2.rs b/src/auth/ed25519/point_avx2.rs index 2dbe0e03..2eb34880 100644 --- a/src/auth/ed25519/point_avx2.rs +++ b/src/auth/ed25519/point_avx2.rs @@ -666,8 +666,8 @@ impl ExtendedPointIfma { /// Double this point using HWCD'08 parallel doubling. /// - /// Uses the reduced-input `square()` + `mul()` path: reduces before each - /// multiply to eliminate overflow corrections (~40% fewer ops per double). + /// Uses the reduced-input `square()` + `mul()` path, reducing before each + /// multiply so the multiply need not handle overflow corrections. /// /// # Safety /// @@ -1019,10 +1019,8 @@ unsafe fn add_wnaf_digit_ifma_raw(acc: ExtendedPointIfma, table: &[[[i64; 4]; 5] /// wNAF-based Straus/Shamir: `[s]B + [h]A`. /// -/// Uses wNAF(8) for the basepoint scalar `s` (64-entry odd-multiples table, -/// ~28 additions) and wNAF(5) for the public-key scalar `h` (8-entry -/// odd-multiples table, ~43 additions). Total ~71 additions vs ~128 with -/// the radix-16 approach — a ~45% reduction in point additions. +/// Uses wNAF(8) for the basepoint scalar `s` (64-entry odd-multiples table) +/// and wNAF(5) for the public-key scalar `h` (8-entry odd-multiples table). /// /// The loop scans 256 bit positions with 1 double per bit (same 256 total /// doublings), but additions are much sparser due to the wNAF non-adjacency @@ -1043,8 +1041,7 @@ pub(crate) unsafe fn straus_wnaf_vartime_ifma(s: &[u8; 32], h: &[u8; 32], a: &Ex // wNAF(8) basepoint table: [B, 3B, 5B, ..., 127B] (64 entries). // Static compile-time data in .rodata — zero build cost, zero copy. - // Entries loaded on demand via _mm256_loadu_si256 (~5 loads per access, - // only ~28 entries touched per verify due to wNAF sparsity). + // Entries are loaded on demand; wNAF sparsity avoids scanning the table. let base_table = &basepoint_table_ifma::BASEPOINT_WNAF8_IFMA_RAW; // Build wNAF(5) public-key table: [A, 3A, 5A, ..., 15A] (8 entries). diff --git a/src/auth/hkdf.rs b/src/auth/hkdf.rs index ea764ce4..7c2c29be 100644 --- a/src/auth/hkdf.rs +++ b/src/auth/hkdf.rs @@ -184,15 +184,14 @@ impl HkdfSha256 { } let compress = self.compress; - let inner_init = self.inner_init; - let outer_init = self.outer_init; + let mut inner_block = [0u8; SHA256_BLOCK_SIZE]; let mut outer_block = [0u8; SHA256_BLOCK_SIZE]; outer_block[SHA256_OUTPUT_SIZE] = 0x80; outer_block[56..SHA256_BLOCK_SIZE].copy_from_slice(&768u64.to_be_bytes()); let mut inner_hash = [0u8; SHA256_OUTPUT_SIZE]; - let mut state = [0u32; 8]; + let mut state = self.inner_init; let mut counter: u8 = 1; let mut prev_tag = [0u8; SHA256_OUTPUT_SIZE]; @@ -201,10 +200,18 @@ impl HkdfSha256 { return Ok(()); }; - expand_hmac_sha256_inner(compress, &inner_init, None, info, counter, &mut state, &mut inner_hash); + expand_hmac_sha256_inner( + compress, + None, + info, + counter, + &mut state, + &mut inner_hash, + &mut inner_block, + ); expand_hmac_sha256_outer( compress, - &outer_init, + &self.outer_init, &inner_hash, &mut state, &mut outer_block, @@ -214,18 +221,19 @@ impl HkdfSha256 { counter = counter.wrapping_add(1); for chunk in chunks { + state = self.inner_init; expand_hmac_sha256_inner( compress, - &inner_init, Some(&prev_tag), info, counter, &mut state, &mut inner_hash, + &mut inner_block, ); expand_hmac_sha256_outer( compress, - &outer_init, + &self.outer_init, &inner_hash, &mut state, &mut outer_block, @@ -237,6 +245,7 @@ impl HkdfSha256 { ct::zeroize_no_fence(&mut prev_tag); ct::zeroize_no_fence(&mut inner_hash); + ct::zeroize_no_fence(&mut inner_block); ct::zeroize_no_fence(&mut outer_block); for word in state.iter_mut() { // SAFETY: word is a valid, aligned, dereferenceable pointer to initialized memory. @@ -315,6 +324,8 @@ impl HkdfSha256 { } #[cfg(feature = "diag")] +#[unsafe(no_mangle)] +#[inline(never)] pub fn diag_hkdf_sha256_derive_portable(input_key_material: &[u8; SHA256_OUTPUT_SIZE]) -> [u8; SHA256_OUTPUT_SIZE] { let compress = crate::hashes::crypto::sha256::kernels::compress_blocks_fn( crate::hashes::crypto::sha256::kernels::Sha256KernelId::Portable, @@ -428,14 +439,13 @@ impl HkdfSha384 { } let compress = self.compress; - let inner_init = self.inner_init; - let outer_init = self.outer_init; + let mut inner_block = [0u8; SHA384_BLOCK_SIZE]; let mut outer_block = [0u8; SHA384_BLOCK_SIZE]; outer_block[SHA384_OUTPUT_SIZE] = 0x80; outer_block[112..SHA384_BLOCK_SIZE].copy_from_slice(&1408u128.to_be_bytes()); - let mut state = [0u64; 8]; + let mut state = self.inner_init; let mut counter: u8 = 1; let mut prev_tag = [0u8; SHA384_OUTPUT_SIZE]; @@ -444,27 +454,37 @@ impl HkdfSha384 { return Ok(()); }; - expand_hmac_sha384_inner(compress, &inner_init, None, info, counter, &mut state, &mut outer_block); - expand_hmac_sha384_outer(compress, &outer_init, &mut state, &mut outer_block, &mut prev_tag); + expand_hmac_sha384_inner( + compress, + None, + info, + counter, + &mut state, + &mut outer_block, + &mut inner_block, + ); + expand_hmac_sha384_outer(compress, &self.outer_init, &mut state, &mut outer_block, &mut prev_tag); first.copy_from_slice(&prev_tag[..first.len()]); counter = counter.wrapping_add(1); for chunk in chunks { + state = self.inner_init; expand_hmac_sha384_inner( compress, - &inner_init, Some(&prev_tag), info, counter, &mut state, &mut outer_block, + &mut inner_block, ); - expand_hmac_sha384_outer(compress, &outer_init, &mut state, &mut outer_block, &mut prev_tag); + expand_hmac_sha384_outer(compress, &self.outer_init, &mut state, &mut outer_block, &mut prev_tag); chunk.copy_from_slice(&prev_tag[..chunk.len()]); counter = counter.wrapping_add(1); } ct::zeroize_no_fence(&mut prev_tag); + ct::zeroize_no_fence(&mut inner_block); ct::zeroize_no_fence(&mut outer_block); for word in state.iter_mut() { // SAFETY: word is a valid, aligned, dereferenceable pointer to initialized memory. @@ -543,6 +563,8 @@ impl HkdfSha384 { } #[cfg(feature = "diag")] +#[unsafe(no_mangle)] +#[inline(never)] pub fn diag_hkdf_sha384_derive_portable(input_key_material: &[u8; SHA384_OUTPUT_SIZE]) -> [u8; SHA384_OUTPUT_SIZE] { let compress = crate::hashes::crypto::sha384::kernels::compress_blocks_fn( crate::hashes::crypto::sha384::kernels::Sha384KernelId::Portable, @@ -637,14 +659,13 @@ impl HkdfSha512 { } let compress = self.compress; - let inner_init = self.inner_init; - let outer_init = self.outer_init; + let mut inner_block = [0u8; SHA512_BLOCK_SIZE]; let mut outer_block = [0u8; SHA512_BLOCK_SIZE]; outer_block[SHA512_OUTPUT_SIZE] = 0x80; outer_block[112..SHA512_BLOCK_SIZE].copy_from_slice(&1536u128.to_be_bytes()); - let mut state = [0u64; 8]; + let mut state = self.inner_init; let mut counter: u8 = 1; let mut prev_tag = [0u8; SHA512_OUTPUT_SIZE]; @@ -653,27 +674,37 @@ impl HkdfSha512 { return Ok(()); }; - expand_hmac_sha512_inner(compress, &inner_init, None, info, counter, &mut state, &mut outer_block); - expand_hmac_sha512_outer(compress, &outer_init, &mut state, &mut outer_block, &mut prev_tag); + expand_hmac_sha512_inner( + compress, + None, + info, + counter, + &mut state, + &mut outer_block, + &mut inner_block, + ); + expand_hmac_sha512_outer(compress, &self.outer_init, &mut state, &mut outer_block, &mut prev_tag); first.copy_from_slice(&prev_tag[..first.len()]); counter = counter.wrapping_add(1); for chunk in chunks { + state = self.inner_init; expand_hmac_sha512_inner( compress, - &inner_init, Some(&prev_tag), info, counter, &mut state, &mut outer_block, + &mut inner_block, ); - expand_hmac_sha512_outer(compress, &outer_init, &mut state, &mut outer_block, &mut prev_tag); + expand_hmac_sha512_outer(compress, &self.outer_init, &mut state, &mut outer_block, &mut prev_tag); chunk.copy_from_slice(&prev_tag[..chunk.len()]); counter = counter.wrapping_add(1); } ct::zeroize_no_fence(&mut prev_tag); + ct::zeroize_no_fence(&mut inner_block); ct::zeroize_no_fence(&mut outer_block); for word in state.iter_mut() { // SAFETY: word is a valid, aligned, dereferenceable pointer to initialized memory. @@ -752,6 +783,8 @@ impl HkdfSha512 { } #[cfg(feature = "diag")] +#[unsafe(no_mangle)] +#[inline(never)] pub fn diag_hkdf_sha512_derive_portable(input_key_material: &[u8; SHA512_OUTPUT_SIZE]) -> [u8; SHA512_OUTPUT_SIZE] { let compress = crate::hashes::crypto::sha512::kernels::compress_blocks_fn( crate::hashes::crypto::sha512::kernels::Sha512KernelId::Portable, @@ -776,20 +809,18 @@ impl Drop for HkdfSha512 { #[allow(clippy::indexing_slicing)] fn expand_hmac_sha256_inner( compress: Sha256CompressBlocksFn, - inner_init: &[u32; 8], prev: Option<&[u8; SHA256_OUTPUT_SIZE]>, info: &[u8], counter: u8, state: &mut [u32; 8], out: &mut [u8; SHA256_OUTPUT_SIZE], + block: &mut [u8; SHA256_BLOCK_SIZE], ) { - *state = *inner_init; - let prev_len = if prev.is_some() { SHA256_OUTPUT_SIZE } else { 0 }; let msg_len = prev_len.strict_add(info.len()).strict_add(1); let total_bytes = (SHA256_BLOCK_SIZE as u64).strict_add(msg_len as u64); - let mut block = [0u8; SHA256_BLOCK_SIZE]; + block.fill(0); let mut pos = 0usize; if let Some(prev) = prev { @@ -806,8 +837,8 @@ fn expand_hmac_sha256_inner( pos = pos.strict_add(take); info_off = info_off.strict_add(take); if pos == SHA256_BLOCK_SIZE { - compress_hkdf_sha256_block(compress, state, &block); - block = [0u8; SHA256_BLOCK_SIZE]; + compress_hkdf_sha256_block(compress, state, block); + block.fill(0); pos = 0; } } @@ -815,18 +846,18 @@ fn expand_hmac_sha256_inner( block[pos] = counter; pos = pos.strict_add(1); if pos == SHA256_BLOCK_SIZE { - compress_hkdf_sha256_block(compress, state, &block); - block = [0u8; SHA256_BLOCK_SIZE]; + compress_hkdf_sha256_block(compress, state, block); + block.fill(0); pos = 0; } block[pos] = 0x80; if pos.strict_add(1) > 56 { - compress_hkdf_sha256_block(compress, state, &block); - block = [0u8; SHA256_BLOCK_SIZE]; + compress_hkdf_sha256_block(compress, state, block); + block.fill(0); } block[56..SHA256_BLOCK_SIZE].copy_from_slice(&total_bytes.strict_mul(8).to_be_bytes()); - compress_hkdf_sha256_block(compress, state, &block); + compress_hkdf_sha256_block(compress, state, block); write_u32x8_be(out, state); } @@ -852,20 +883,18 @@ fn expand_hmac_sha256_outer( #[allow(clippy::indexing_slicing)] fn expand_hmac_sha384_inner( compress: Sha384CompressBlocksFn, - inner_init: &[u64; 8], prev: Option<&[u8; SHA384_OUTPUT_SIZE]>, info: &[u8], counter: u8, state: &mut [u64; 8], outer_block: &mut [u8; SHA384_BLOCK_SIZE], + block: &mut [u8; SHA384_BLOCK_SIZE], ) { - *state = *inner_init; - let prev_len = if prev.is_some() { SHA384_OUTPUT_SIZE } else { 0 }; let msg_len = prev_len.strict_add(info.len()).strict_add(1); let total_bytes = (SHA384_BLOCK_SIZE as u128).strict_add(msg_len as u128); - let mut block = [0u8; SHA384_BLOCK_SIZE]; + block.fill(0); let mut pos = 0usize; if let Some(prev) = prev { @@ -882,8 +911,8 @@ fn expand_hmac_sha384_inner( pos = pos.strict_add(take); info_off = info_off.strict_add(take); if pos == SHA384_BLOCK_SIZE { - compress(state, &block); - block = [0u8; SHA384_BLOCK_SIZE]; + compress(state, block); + block.fill(0); pos = 0; } } @@ -891,18 +920,18 @@ fn expand_hmac_sha384_inner( block[pos] = counter; pos = pos.strict_add(1); if pos == SHA384_BLOCK_SIZE { - compress(state, &block); - block = [0u8; SHA384_BLOCK_SIZE]; + compress(state, block); + block.fill(0); pos = 0; } block[pos] = 0x80; if pos.strict_add(1) > 112 { - compress(state, &block); - block = [0u8; SHA384_BLOCK_SIZE]; + compress(state, block); + block.fill(0); } block[112..SHA384_BLOCK_SIZE].copy_from_slice(&total_bytes.strict_mul(8).to_be_bytes()); - compress(state, &block); + compress(state, block); write_u64x6_be(&mut outer_block[..SHA384_OUTPUT_SIZE], state); } @@ -926,20 +955,18 @@ fn expand_hmac_sha384_outer( #[allow(clippy::indexing_slicing)] fn expand_hmac_sha512_inner( compress: Sha512CompressBlocksFn, - inner_init: &[u64; 8], prev: Option<&[u8; SHA512_OUTPUT_SIZE]>, info: &[u8], counter: u8, state: &mut [u64; 8], outer_block: &mut [u8; SHA512_BLOCK_SIZE], + block: &mut [u8; SHA512_BLOCK_SIZE], ) { - *state = *inner_init; - let prev_len = if prev.is_some() { SHA512_OUTPUT_SIZE } else { 0 }; let msg_len = prev_len.strict_add(info.len()).strict_add(1); let total_bytes = (SHA512_BLOCK_SIZE as u128).strict_add(msg_len as u128); - let mut block = [0u8; SHA512_BLOCK_SIZE]; + block.fill(0); let mut pos = 0usize; if let Some(prev) = prev { @@ -956,8 +983,8 @@ fn expand_hmac_sha512_inner( pos = pos.strict_add(take); info_off = info_off.strict_add(take); if pos == SHA512_BLOCK_SIZE { - compress(state, &block); - block = [0u8; SHA512_BLOCK_SIZE]; + compress(state, block); + block.fill(0); pos = 0; } } @@ -965,18 +992,18 @@ fn expand_hmac_sha512_inner( block[pos] = counter; pos = pos.strict_add(1); if pos == SHA512_BLOCK_SIZE { - compress(state, &block); - block = [0u8; SHA512_BLOCK_SIZE]; + compress(state, block); + block.fill(0); pos = 0; } block[pos] = 0x80; if pos.strict_add(1) > 112 { - compress(state, &block); - block = [0u8; SHA512_BLOCK_SIZE]; + compress(state, block); + block.fill(0); } block[112..SHA512_BLOCK_SIZE].copy_from_slice(&total_bytes.strict_mul(8).to_be_bytes()); - compress(state, &block); + compress(state, block); write_u64x8_be(&mut outer_block[..SHA512_OUTPUT_SIZE], state); } diff --git a/src/auth/hmac.rs b/src/auth/hmac.rs index c3b664d1..7cc104ff 100644 --- a/src/auth/hmac.rs +++ b/src/auth/hmac.rs @@ -508,8 +508,8 @@ impl Mac for HmacSha256 { #[allow(clippy::indexing_slicing)] // All indices bounded by prior length checks + fixed-size arrays. fn mac(key: &[u8], data: &[u8]) -> Self::Tag { if hmac_sha256_oneshot_should_stream() { - // Sapphire Rapids regresses badly on the fused stack-buffer shape; the - // public streaming path keeps the same semantics with the measured fast call shape. + // The Sapphire Rapids policy uses the public streaming path instead of + // the fused stack-buffer shape. let mut mac = Self::new(key); mac.update(data); return mac.finalize(); diff --git a/src/auth/kmac.rs b/src/auth/kmac.rs index a386bf52..c2ea51ad 100644 --- a/src/auth/kmac.rs +++ b/src/auth/kmac.rs @@ -13,7 +13,7 @@ use crate::{ }; macro_rules! define_kmac { - ($name:ident, $cshake:ident, $bits:literal) => { + ($name:ident, $cshake:ident, $bits:literal, $min_auth_tag_size:literal) => { #[doc = concat!("KMAC", $bits, " keyed state.")] /// KMAC is a variable-output MAC/PRF. It intentionally does not implement /// [`crate::traits::Mac`], which assumes a fixed-size tag. @@ -29,6 +29,15 @@ macro_rules! define_kmac { } impl $name { + /// Minimum tag length accepted by the authentication verification APIs. + /// + /// This preserves the effective security strength named by the KMAC + /// variant. Protocol primitives that deliberately use shorter outputs + /// must call [`Self::verify_primitive`] or + /// [`Self::verify_tag_primitive`] and enforce their own forgery-attempt + /// limit. + pub const MIN_AUTH_TAG_SIZE: usize = $min_auth_tag_size; + /// Construct a new KMAC state keyed by `key` and domain-separated by /// `customization`. #[must_use] @@ -88,9 +97,16 @@ macro_rules! define_kmac { out } - /// Verify `expected` after traversing its public-length contents. + /// Verify an authentication tag after traversing its public-length + /// contents. /// - /// This is the one-shot helper. Use [`Self::verify`] for an already-accumulated state. + /// Tags shorter than [`Self::MIN_AUTH_TAG_SIZE`] are rejected before + /// KMAC computation. Use [`Self::verify_tag_primitive`] only when a + /// protocol deliberately specifies a shorter output and independently + /// bounds failed verification attempts. + /// + /// This is the one-shot helper. Use [`Self::verify`] for an + /// already-accumulated state. /// Generated-code timing claims are configuration- and release-evidence-bound; /// see `ct.toml`. #[must_use = "MAC verification must be checked; a dropped Result silently accepts a forged tag"] @@ -105,17 +121,49 @@ macro_rules! define_kmac { state.verify(expected) } - #[doc = concat!("Verify `expected` against the current KMAC", $bits, " output after a full public-length comparison.")] + /// Verify an arbitrary nonempty protocol-defined KMAC output. + /// + /// This primitive does not enforce [`Self::MIN_AUTH_TAG_SIZE`]. + /// Authentication protocols using shorter values must set an explicit + /// forgery budget and bound failed verification attempts. + #[must_use = "MAC verification must be checked; a dropped Result silently accepts a forged tag"] + pub fn verify_tag_primitive( + key: &[u8], + customization: &[u8], + data: &[u8], + expected: &[u8], + ) -> Result<(), VerificationError> { + let mut state = Self::new(key, customization); + state.update(data); + state.verify_primitive(expected) + } + + #[doc = concat!("Verify an authentication tag against the current KMAC", $bits, " output after a full public-length comparison.")] /// This checks the MAC for the bytes already absorbed into `self`; it does /// not recompute from `(key, customization, data)` like [`Self::verify_tag`]. + /// Tags shorter than [`Self::MIN_AUTH_TAG_SIZE`] are rejected before + /// KMAC computation. /// Generated-code timing claims are configuration- and release-evidence-bound; /// see `ct.toml`. #[must_use = "MAC verification must be checked; a dropped Result silently accepts a forged tag"] pub fn verify(&self, expected: &[u8]) -> Result<(), VerificationError> { - if expected.is_empty() { + if expected.len() < Self::MIN_AUTH_TAG_SIZE { return Err(VerificationError::new()); } + self.verify_primitive(expected) + } + /// Verify an arbitrary nonempty protocol-defined output against the + /// current KMAC state. + /// + /// This primitive does not enforce [`Self::MIN_AUTH_TAG_SIZE`]. + /// Authentication protocols using shorter values must set an explicit + /// forgery budget and bound failed verification attempts. + #[must_use = "MAC verification must be checked; a dropped Result silently accepts a forged tag"] + pub fn verify_primitive(&self, expected: &[u8]) -> Result<(), VerificationError> { + if expected.is_empty() { + return Err(VerificationError::new()); + } let mut reader = self.finalize_reader(expected.len()); let mut diff = 0u8; let mut block = [0u8; 64]; @@ -136,8 +184,8 @@ macro_rules! define_kmac { }; } -define_kmac!(Kmac128, Cshake128, "128"); -define_kmac!(Kmac256, Cshake256, "256"); +define_kmac!(Kmac128, Cshake128, "128", 16); +define_kmac!(Kmac256, Cshake256, "256", 32); #[cfg(test)] mod tests { @@ -164,4 +212,39 @@ mod tests { kmac128.finalize_into(&mut actual); assert_eq!(actual, expected128); } + + #[test] + fn authentication_verification_enforces_variant_strength() { + const KEY: &[u8] = b"authentication-policy-key"; + const CUSTOMIZATION: &[u8] = b"protocol=v1"; + const MESSAGE: &[u8] = b"authenticated message"; + + for len in [ + Kmac128::MIN_AUTH_TAG_SIZE - 1, + Kmac128::MIN_AUTH_TAG_SIZE, + Kmac128::MIN_AUTH_TAG_SIZE + 1, + ] { + let mut tag = [0u8; Kmac128::MIN_AUTH_TAG_SIZE + 1]; + Kmac128::mac_into(KEY, CUSTOMIZATION, MESSAGE, &mut tag[..len]); + assert_eq!( + Kmac128::verify_tag(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]).is_ok(), + len >= Kmac128::MIN_AUTH_TAG_SIZE + ); + assert!(Kmac128::verify_tag_primitive(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]).is_ok()); + } + + for len in [ + Kmac256::MIN_AUTH_TAG_SIZE - 1, + Kmac256::MIN_AUTH_TAG_SIZE, + Kmac256::MIN_AUTH_TAG_SIZE + 1, + ] { + let mut tag = [0u8; Kmac256::MIN_AUTH_TAG_SIZE + 1]; + Kmac256::mac_into(KEY, CUSTOMIZATION, MESSAGE, &mut tag[..len]); + assert_eq!( + Kmac256::verify_tag(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]).is_ok(), + len >= Kmac256::MIN_AUTH_TAG_SIZE + ); + assert!(Kmac256::verify_tag_primitive(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]).is_ok()); + } + } } diff --git a/src/auth/mlkem/portable.rs b/src/auth/mlkem/portable.rs index dbfae856..50865676 100644 --- a/src/auth/mlkem/portable.rs +++ b/src/auth/mlkem/portable.rs @@ -52,9 +52,14 @@ use core::arch::x86_64::{ _mm256_srli_epi32, _mm256_storeu_si256, _mm256_sub_epi16, _mm256_sub_epi32, }; +#[cfg(test)] +use crate::hashes::crypto::Shake256; use crate::{ auth::mlkem::MlKemError, - hashes::crypto::{Sha3_256, Sha3_512, Shake128, Shake128XofReader, Shake256, Shake256XofReader}, + hashes::crypto::{ + Sha3_256, Shake128, Shake128XofReader, + sha3::{MlKemShake256XofReader, mlkem_sha3_512_digest, mlkem_shake256_two_part_into}, + }, traits::{ Digest, Xof, ct::{self}, @@ -4321,10 +4326,10 @@ fn sample_noise_pair( nonce1: u8, out1: &mut Poly, ) { - let (mut reader0, mut reader1) = Shake256::xof_seeded_32_1_pair(seed, nonce0, nonce1); + let (mut reader0, mut reader1) = MlKemShake256XofReader::seeded_32_1_pair(seed, nonce0, nonce1); let mut buf0 = [0u8; RANDOM_BYTES]; let mut buf1 = [0u8; RANDOM_BYTES]; - Shake256XofReader::squeeze_pair(&mut reader0, &mut reader1, &mut buf0, &mut buf1); + MlKemShake256XofReader::squeeze_pair(&mut reader0, &mut reader1, &mut buf0, &mut buf1); match RANDOM_BYTES { ETA2_RANDOM_BYTES => { sample_poly_cbd_eta2(&buf0, out0); @@ -4353,12 +4358,12 @@ fn sample_noise_quad( out3: &mut Poly, ) { let (mut reader0, mut reader1, mut reader2, mut reader3) = - Shake256::xof_seeded_32_1_quad(seed, nonce0, nonce1, nonce2, nonce3); + MlKemShake256XofReader::seeded_32_1_quad(seed, nonce0, nonce1, nonce2, nonce3); let mut buf0 = [0u8; RANDOM_BYTES]; let mut buf1 = [0u8; RANDOM_BYTES]; let mut buf2 = [0u8; RANDOM_BYTES]; let mut buf3 = [0u8; RANDOM_BYTES]; - Shake256XofReader::squeeze_quad( + MlKemShake256XofReader::squeeze_quad( &mut reader0, &mut reader1, &mut reader2, @@ -7538,21 +7543,17 @@ fn h(input: &[u8]) -> [u8; HASH_BYTES] { } fn g(input: &[u8]) -> [u8; 64] { - Sha3_512::digest(input) + mlkem_sha3_512_digest(input) } fn j(z: &[u8], c: &[u8]) -> [u8; SHARED_SECRET_BYTES] { - let mut xof = Shake256::new(); - xof.update(z); - xof.update(c); - let mut reader = xof.finalize_xof(); let mut out = [0u8; SHARED_SECRET_BYTES]; - reader.squeeze(&mut out); + mlkem_shake256_two_part_into(z, c, &mut out); out } fn prf_eta(seed: &[u8; SEED_BYTES], nonce: u8, out: &mut [u8; RANDOM_BYTES]) { - let mut reader = Shake256::xof_seeded_32_1(seed, nonce); + let mut reader = MlKemShake256XofReader::seeded_32_1(seed, nonce); reader.squeeze(out); } diff --git a/src/auth/pbkdf2.rs b/src/auth/pbkdf2.rs index 839354e4..c55bab03 100644 --- a/src/auth/pbkdf2.rs +++ b/src/auth/pbkdf2.rs @@ -123,6 +123,8 @@ pub enum Pbkdf2Error { SaltTooShort, /// The requested output length exceeds `(2^32 − 1) × hLen`. OutputTooLong, + /// The iteration count exceeds the selected password-verification work limit. + ExcessiveIterations, } impl fmt::Display for Pbkdf2Error { @@ -130,6 +132,7 @@ impl fmt::Display for Pbkdf2Error { match self { Self::InvalidIterations => f.write_str("PBKDF2 iteration count must be at least 1"), Self::WeakIterations => f.write_str("PBKDF2 iteration count is below the password policy minimum"), + Self::ExcessiveIterations => f.write_str("PBKDF2 iteration count exceeds the password policy maximum"), Self::SaltTooShort => f.write_str("PBKDF2 salt is below the password policy minimum"), Self::OutputTooLong => f.write_str("PBKDF2 output length exceeds algorithm maximum"), } @@ -183,8 +186,10 @@ impl<'a> Pbkdf2Params<'a> { /// /// PBKDF2 has legitimate low-iteration test vectors, but password storage /// should reject those values. The default one-shot password APIs use the -/// type-specific policy constants; use explicit policies only for migrations -/// or deployments with stricter local requirements. +/// type-specific policy constants and a type-specific verification work limit. +/// Use explicit lower-bound policies only for migrations or deployments with +/// stricter local requirements; use the bounded policy methods when an +/// attacker can select the stored iteration count. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Pbkdf2VerifyPolicy { /// Minimum accepted iteration count. @@ -209,6 +214,13 @@ impl Pbkdf2VerifyPolicy { params.iterations() >= self.min_iterations && params.salt().len() >= self.min_salt_len } + /// Return `true` when `params` satisfies this policy and the explicit + /// verification work limit. + #[must_use] + pub const fn allows_bounded(&self, params: &Pbkdf2Params<'_>, max_iterations: u32) -> bool { + self.allows(params) && params.iterations() <= max_iterations + } + fn check(&self, params: &Pbkdf2Params<'_>) -> Result<(), Pbkdf2Error> { if params.iterations() < self.min_iterations { return Err(Pbkdf2Error::WeakIterations); @@ -218,6 +230,14 @@ impl Pbkdf2VerifyPolicy { } Ok(()) } + + fn check_bounded(&self, params: &Pbkdf2Params<'_>, max_iterations: u32) -> Result<(), Pbkdf2Error> { + self.check(params)?; + if params.iterations() > max_iterations { + return Err(Pbkdf2Error::ExcessiveIterations); + } + Ok(()) + } } macro_rules! define_pbkdf2_sha2 { @@ -257,6 +277,12 @@ macro_rules! define_pbkdf2_sha2 { pub const OUTPUT_SIZE: usize = $output_size_const; /// Minimum iteration count recommended for compliance-sensitive deployments. pub const MIN_RECOMMENDED_ITERATIONS: u32 = $recommended_iterations; + /// Maximum iteration count accepted by default password-verification APIs. + /// + /// Raw PBKDF2 derivation and primitive verification remain unbounded for + /// protocol compatibility. Stored-password verification rejects larger + /// attacker-controlled work before constructing HMAC state. + pub const MAX_VERIFY_ITERATIONS: u32 = $recommended_iterations * 10; /// Minimum salt length (bytes) recommended for compliance-sensitive deployments. pub const MIN_SALT_LEN: usize = 16; /// Default password-hashing verification policy for this PBKDF2 variant. @@ -273,7 +299,11 @@ macro_rules! define_pbkdf2_sha2 { Self::params_with_policy(salt, iterations, &Self::DEFAULT_VERIFY_POLICY) } - /// Build PBKDF2 password parameters under an explicit policy. + /// Build PBKDF2 password parameters under an explicit lower-bound policy. + /// + /// This constructor does not impose an upper work limit. Use + /// [`params_with_policy_bounded`](Self::params_with_policy_bounded) when + /// the iteration count comes from an untrusted password record. /// /// # Errors /// @@ -289,6 +319,24 @@ macro_rules! define_pbkdf2_sha2 { Ok(params) } + /// Build PBKDF2 password parameters under an explicit lower-bound policy + /// and verification work limit. + /// + /// # Errors + /// + /// Returns [`Pbkdf2Error`] when the iteration count is zero, the supplied + /// policy rejects the parameters, or `iterations > max_iterations`. + pub fn params_with_policy_bounded<'a>( + salt: &'a [u8], + iterations: u32, + policy: &Pbkdf2VerifyPolicy, + max_iterations: u32, + ) -> Result, Pbkdf2Error> { + let params = Pbkdf2Params::new_primitive(salt, iterations)?; + policy.check_bounded(¶ms, max_iterations)?; + Ok(params) + } + /// Pre-compute HMAC prefix states from `password`. #[must_use] #[allow(clippy::indexing_slicing)] // password.len() <= block size in the else branch. @@ -425,11 +473,21 @@ macro_rules! define_pbkdf2_sha2 { #[allow(clippy::indexing_slicing)] #[must_use = "password verification must be checked; a dropped Result silently accepts the wrong password"] pub fn verify(&self, salt: &[u8], iterations: u32, expected: &[u8]) -> Result<(), VerificationError> { - self.verify_with_policy(salt, iterations, expected, &Self::DEFAULT_VERIFY_POLICY) + self.verify_with_policy_bounded( + salt, + iterations, + expected, + &Self::DEFAULT_VERIFY_POLICY, + Self::MAX_VERIFY_ITERATIONS, + ) } /// Verify `expected` after a full public-length comparison using an - /// explicit password policy. + /// explicit lower-bound password policy. + /// + /// This compatibility method does not impose an upper work limit. Use + /// [`verify_with_policy_bounded`](Self::verify_with_policy_bounded) when + /// the iteration count comes from an untrusted password record. /// /// Generated-code timing claims are configuration- and release-evidence-bound; /// see `ct.toml`. @@ -442,12 +500,36 @@ macro_rules! define_pbkdf2_sha2 { expected: &[u8], policy: &Pbkdf2VerifyPolicy, ) -> Result<(), VerificationError> { - let Ok(params) = Self::params_with_policy(salt, iterations, policy) else { - return Err(VerificationError::new()); - }; + let params = Self::params_with_policy(salt, iterations, policy).map_err(|_| VerificationError::new())?; self.verify_primitive(params.salt(), params.iterations(), expected) } + /// Verify `expected` under an explicit lower-bound policy and + /// caller-selected verification work limit. + #[allow(clippy::indexing_slicing)] + #[must_use = "password verification must be checked; a dropped Result silently accepts the wrong password"] + pub fn verify_with_policy_bounded( + &self, + salt: &[u8], + iterations: u32, + expected: &[u8], + policy: &Pbkdf2VerifyPolicy, + max_iterations: u32, + ) -> Result<(), VerificationError> { + let params = Self::verification_params_bounded(salt, iterations, policy, max_iterations)?; + self.verify_primitive(params.salt(), params.iterations(), expected) + } + + fn verification_params_bounded<'a>( + salt: &'a [u8], + iterations: u32, + policy: &Pbkdf2VerifyPolicy, + max_iterations: u32, + ) -> Result, VerificationError> { + Self::params_with_policy_bounded(salt, iterations, policy, max_iterations) + .map_err(|_| VerificationError::new()) + } + /// Verify `expected` against the derived key without password policy checks. /// /// This is the primitive/test-vector verification path. Stored password @@ -571,10 +653,22 @@ macro_rules! define_pbkdf2_sha2 { iterations: u32, expected: &[u8], ) -> Result<(), VerificationError> { - Self::verify_password_with_policy(password, salt, iterations, expected, &Self::DEFAULT_VERIFY_POLICY) + Self::verify_password_with_policy_bounded( + password, + salt, + iterations, + expected, + &Self::DEFAULT_VERIFY_POLICY, + Self::MAX_VERIFY_ITERATIONS, + ) } - /// Verify a password in one shot using an explicit password policy. + /// Verify a password in one shot using an explicit lower-bound password + /// policy. + /// + /// This compatibility method does not impose an upper work limit. Use + /// [`verify_password_with_policy_bounded`](Self::verify_password_with_policy_bounded) + /// when the iteration count comes from an untrusted password record. #[inline] #[must_use = "password verification must be checked; a dropped Result silently accepts the wrong password"] pub fn verify_password_with_policy( @@ -584,7 +678,24 @@ macro_rules! define_pbkdf2_sha2 { expected: &[u8], policy: &Pbkdf2VerifyPolicy, ) -> Result<(), VerificationError> { - Self::new(password).verify_with_policy(salt, iterations, expected, policy) + let params = Self::params_with_policy(salt, iterations, policy).map_err(|_| VerificationError::new())?; + Self::new(password).verify_primitive(params.salt(), params.iterations(), expected) + } + + /// Verify a password in one shot under an explicit lower-bound policy + /// and caller-selected verification work limit. + #[inline] + #[must_use = "password verification must be checked; a dropped Result silently accepts the wrong password"] + pub fn verify_password_with_policy_bounded( + password: &[u8], + salt: &[u8], + iterations: u32, + expected: &[u8], + policy: &Pbkdf2VerifyPolicy, + max_iterations: u32, + ) -> Result<(), VerificationError> { + let params = Self::verification_params_bounded(salt, iterations, policy, max_iterations)?; + Self::new(password).verify_primitive(params.salt(), params.iterations(), expected) } /// Verify a password in one shot without password policy checks. @@ -1648,6 +1759,77 @@ mod tests { assert!(Pbkdf2Sha256::derive_key_with_params(b"pw", params, &mut empty).is_ok()); } + #[test] + fn password_verification_limits_attacker_controlled_iterations_before_hmac_work() { + let policy = Pbkdf2VerifyPolicy::new(1, 0); + + for iterations in [ + Pbkdf2Sha256::MAX_VERIFY_ITERATIONS.strict_sub(1), + Pbkdf2Sha256::MAX_VERIFY_ITERATIONS, + ] { + let params = + Pbkdf2Sha256::params_with_policy_bounded(b"", iterations, &policy, Pbkdf2Sha256::MAX_VERIFY_ITERATIONS) + .unwrap(); + assert!(policy.allows_bounded(¶ms, Pbkdf2Sha256::MAX_VERIFY_ITERATIONS)); + } + let excessive_sha256 = Pbkdf2Sha256::MAX_VERIFY_ITERATIONS.strict_add(1); + let lower_bound_only = Pbkdf2Sha256::params_with_policy(b"", excessive_sha256, &policy).unwrap(); + assert!(policy.allows(&lower_bound_only)); + assert!(!policy.allows_bounded(&lower_bound_only, Pbkdf2Sha256::MAX_VERIFY_ITERATIONS)); + assert_eq!( + Pbkdf2Sha256::params_with_policy_bounded(b"", excessive_sha256, &policy, Pbkdf2Sha256::MAX_VERIFY_ITERATIONS,), + Err(Pbkdf2Error::ExcessiveIterations) + ); + + let sha256 = Pbkdf2Sha256::new_with_compress_for_test(b"pw", counting_sha256_compress); + SHA256_VERIFY_BLOCKS.store(0, Ordering::Relaxed); + assert!( + sha256 + .verify_with_policy_bounded( + b"", + excessive_sha256, + &[0], + &policy, + Pbkdf2Sha256::MAX_VERIFY_ITERATIONS, + ) + .is_err() + ); + assert_eq!(SHA256_VERIFY_BLOCKS.load(Ordering::Relaxed), 0); + + for iterations in [ + Pbkdf2Sha512::MAX_VERIFY_ITERATIONS.strict_sub(1), + Pbkdf2Sha512::MAX_VERIFY_ITERATIONS, + ] { + let params = + Pbkdf2Sha512::params_with_policy_bounded(b"", iterations, &policy, Pbkdf2Sha512::MAX_VERIFY_ITERATIONS) + .unwrap(); + assert!(policy.allows_bounded(¶ms, Pbkdf2Sha512::MAX_VERIFY_ITERATIONS)); + } + let excessive_sha512 = Pbkdf2Sha512::MAX_VERIFY_ITERATIONS.strict_add(1); + let lower_bound_only = Pbkdf2Sha512::params_with_policy(b"", excessive_sha512, &policy).unwrap(); + assert!(policy.allows(&lower_bound_only)); + assert!(!policy.allows_bounded(&lower_bound_only, Pbkdf2Sha512::MAX_VERIFY_ITERATIONS)); + assert_eq!( + Pbkdf2Sha512::params_with_policy_bounded(b"", excessive_sha512, &policy, Pbkdf2Sha512::MAX_VERIFY_ITERATIONS,), + Err(Pbkdf2Error::ExcessiveIterations) + ); + + let sha512 = Pbkdf2Sha512::new_with_compress_for_test(b"pw", counting_sha512_compress); + SHA512_VERIFY_BLOCKS.store(0, Ordering::Relaxed); + assert!( + sha512 + .verify_with_policy_bounded( + b"", + excessive_sha512, + &[0], + &policy, + Pbkdf2Sha512::MAX_VERIFY_ITERATIONS, + ) + .is_err() + ); + assert_eq!(SHA512_VERIFY_BLOCKS.load(Ordering::Relaxed), 0); + } + #[test] fn sha512_zero_iterations_error() { let mut dk = [0u8; 64]; diff --git a/src/auth/rsa.rs b/src/auth/rsa.rs index fe33c6ae..deba1df1 100644 --- a/src/auth/rsa.rs +++ b/src/auth/rsa.rs @@ -118,6 +118,10 @@ const PRIVATE_FIXED_WINDOW_TABLE_ENTRIES: usize = 16; const RSA_KEYGEN_PUBLIC_EXPONENT: u64 = 65_537; #[cfg(feature = "getrandom")] const RSA_KEYGEN_MILLER_RABIN_ROUNDS: usize = 32; +const RSA_IMPORT_MILLER_RABIN_BASES: [u16; 32] = [ + 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, + 113, 127, 131, +]; #[cfg(feature = "getrandom")] const RSA_KEYGEN_PAIR_ATTEMPTS: usize = 64; #[cfg(feature = "getrandom")] @@ -136,8 +140,7 @@ const RSA_KEYGEN_DRBG_HMAC_BLOCK_BYTES: usize = 64; const RSA_KEYGEN_DRBG_PERSONALIZATION: &[u8] = b"rscrypto RSA FIPS 186-5 A.1.3 HMAC_DRBG"; #[cfg(feature = "getrandom")] const RSA_KEYGEN_SQRT2_HALF_TOP64: [u8; 8] = [0xb5, 0x04, 0xf3, 0x33, 0xf9, 0xde, 0x64, 0x84]; -#[cfg(feature = "getrandom")] -const RSA_KEYGEN_SMALL_PRIMES: &[u16] = &[ +const RSA_SMALL_PRIMES: &[u16] = &[ 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149, 151, 157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211, 223, 227, 229, 233, 239, 241, 251, 257, 263, 269, 271, 277, 281, 283, 293, 307, 311, 313, 317, 331, 337, 347, 349, 353, 359, 367, 373, 379, 383, @@ -1187,6 +1190,11 @@ pub struct RsaPublicKey { /// Validated RSA private key. /// +/// Imports require two conventional half-modulus-width factors that pass +/// trial division and a 32-base Miller-Rabin probable-prime screen. The +/// screening detects weak factor shapes and compositeness evidence; it is not +/// a mathematical proof of primality. +/// /// Private operations use either OS-backed blinding through the `getrandom` /// feature or an explicit caller-supplied blinding factor and modular inverse /// for deterministic tests and constrained integrations. @@ -1212,8 +1220,7 @@ pub struct RsaPrivateScratch { one: SecretBigEndianBuffer, public_scratch: RsaPublicScratch, mul_scratch: RsaPrivateMulScratch, - exponent_p_scratch: RsaPrivateExponentScratch, - exponent_q_scratch: RsaPrivateExponentScratch, + exponent_scratch: RsaPrivateExponentScratch, } /// RSA signer bound to a concrete signature profile. @@ -1245,7 +1252,9 @@ pub struct RsaJwtVerifier<'a> { /// Borrowed RSA private-key CRT components. /// /// These fields contain private key material. Keep values canonical unsigned -/// big-endian, without leading zero padding. +/// big-endian, without leading zero padding. Both factors must have the +/// conventional half-modulus bit length and pass the private-import +/// probable-prime screen. #[derive(Clone, Copy)] pub struct RsaPrivateKeyParts<'a> { /// RSA modulus `n`. @@ -1326,7 +1335,8 @@ impl RsaPrivateKey { /// # Errors /// /// Returns [`RsaKeyError`] if the DER is malformed or the key components are - /// inconsistent with RSA private-key policy. + /// inconsistent with RSA private-key policy, including factor width or + /// probable-prime validation. pub fn from_pkcs1_der(der: &[u8]) -> Result { Self::from_pkcs1_der_with_policy(der, &RsaPublicKeyPolicy::default()) } @@ -1335,7 +1345,8 @@ impl RsaPrivateKey { /// /// # Errors /// - /// Returns [`RsaKeyError`] if parsing or validation fails. + /// Returns [`RsaKeyError`] if parsing, factor-strength validation, or + /// component consistency validation fails. pub fn from_pkcs1_der_with_policy(der: &[u8], policy: &RsaPublicKeyPolicy) -> Result { parse_pkcs1_private_key_der_with_policy(der, policy).map(|components| Self { components }) } @@ -1350,7 +1361,8 @@ impl RsaPrivateKey { /// # Errors /// /// Returns [`RsaKeyError`] if the DER is malformed, not an RSA private key, - /// or the embedded PKCS #1 private key is invalid. + /// or the embedded PKCS #1 private key fails factor-strength or consistency + /// validation. pub fn from_pkcs8_der(der: &[u8]) -> Result { Self::from_pkcs8_der_with_policy(der, &RsaPublicKeyPolicy::default()) } @@ -1359,7 +1371,8 @@ impl RsaPrivateKey { /// /// # Errors /// - /// Returns [`RsaKeyError`] if parsing or validation fails. + /// Returns [`RsaKeyError`] if parsing, factor-strength validation, or + /// component consistency validation fails. pub fn from_pkcs8_der_with_policy(der: &[u8], policy: &RsaPublicKeyPolicy) -> Result { parse_pkcs8_private_key_der_with_policy(der, policy).map(|components| Self { components }) } @@ -1369,7 +1382,7 @@ impl RsaPrivateKey { /// # Errors /// /// Returns [`RsaKeyError`] if the components violate RSA private-key policy - /// or fail consistency checks. + /// or fail factor-strength or consistency checks. pub fn from_components(parts: RsaPrivateKeyParts<'_>) -> Result { Self::from_components_with_policy(parts, &RsaPublicKeyPolicy::default()) } @@ -1379,7 +1392,7 @@ impl RsaPrivateKey { /// # Errors /// /// Returns [`RsaKeyError`] if the components violate `policy` or fail - /// consistency checks. + /// factor-strength or consistency checks. pub fn from_components_with_policy( parts: RsaPrivateKeyParts<'_>, policy: &RsaPublicKeyPolicy, @@ -2533,8 +2546,7 @@ impl RsaPrivateScratch { one, public_scratch: components.public.public_scratch(), mul_scratch: RsaPrivateMulScratch::new(components.public.modulus.limbs.len()), - exponent_p_scratch: RsaPrivateExponentScratch::new(components.prime_p_modulus.limbs.len()), - exponent_q_scratch: RsaPrivateExponentScratch::new(components.prime_q_modulus.limbs.len()), + exponent_scratch: RsaPrivateExponentScratch::new(components.public.modulus.limbs.len()), } } @@ -2577,8 +2589,7 @@ impl RsaPrivateScratch { ct::zeroize_words(&mut self.public_scratch.limbs[..]); ct::zeroize(&mut self.public_scratch.bytes[..]); self.mul_scratch.clear(); - self.exponent_p_scratch.clear(); - self.exponent_q_scratch.clear(); + self.exponent_scratch.clear(); } } @@ -2778,7 +2789,7 @@ impl RsaPrivateKeyComponents { scratch.blinding_factor.as_slice(), modulus_p, factor_p, - &mut scratch.exponent_p_scratch, + &mut scratch.exponent_scratch, )?; if is_zero_unsigned_be(factor_p) { return Err(RsaPrivateOpError::InvalidBlindingFactor); @@ -2794,7 +2805,7 @@ impl RsaPrivateKeyComponents { scratch.blinding_factor.as_slice(), modulus_q, factor_q, - &mut scratch.exponent_q_scratch, + &mut scratch.exponent_scratch, )?; if is_zero_unsigned_be(factor_q) { return Err(RsaPrivateOpError::InvalidBlindingFactor); @@ -2837,7 +2848,7 @@ impl RsaPrivateKeyComponents { .as_mut_slice() .get_mut(..prime_p.len()) .ok_or(RsaPrivateOpError::InvalidScratch)?, - &mut scratch.exponent_p_scratch, + &mut scratch.exponent_scratch, )?; private_exponentiate_representative_with_scratch( modulus_q, @@ -2856,7 +2867,7 @@ impl RsaPrivateKeyComponents { .as_mut_slice() .get_mut(..prime_q.len()) .ok_or(RsaPrivateOpError::InvalidScratch)?, - &mut scratch.exponent_q_scratch, + &mut scratch.exponent_scratch, )?; private_import_unsigned_be_mod_to_fixed( @@ -2871,7 +2882,7 @@ impl RsaPrivateKeyComponents { .as_mut_slice() .get_mut(..prime_p.len()) .ok_or(RsaPrivateOpError::InvalidScratch)?, - &mut scratch.exponent_p_scratch, + &mut scratch.exponent_scratch, )?; private_sub_mod_unsigned_be_to_fixed( scratch @@ -3516,8 +3527,9 @@ impl RsaPrivateKeyComponents { let prime_q = self.prime_q.as_bytes(); let modulus_p = &self.prime_p_modulus; let modulus_q = &self.prime_q_modulus; - scratch.exponent_p_scratch.ensure_limb_count(modulus_p.limbs.len())?; - scratch.exponent_q_scratch.ensure_limb_count(modulus_q.limbs.len())?; + scratch + .exponent_scratch + .ensure_limb_count(core::cmp::max(modulus_p.limbs.len(), modulus_q.limbs.len()))?; { let representative_p = scratch @@ -3529,7 +3541,7 @@ impl RsaPrivateKeyComponents { scratch.blinded.as_slice(), modulus_p, representative_p, - &mut scratch.exponent_p_scratch, + &mut scratch.exponent_scratch, )?; } { @@ -3542,7 +3554,7 @@ impl RsaPrivateKeyComponents { scratch.blinded.as_slice(), modulus_q, representative_q, - &mut scratch.exponent_q_scratch, + &mut scratch.exponent_scratch, )?; } @@ -3559,7 +3571,7 @@ impl RsaPrivateKeyComponents { .as_mut_slice() .get_mut(..prime_p.len()) .ok_or(RsaPrivateOpError::InvalidScratch)?, - &mut scratch.exponent_p_scratch, + &mut scratch.exponent_scratch, )?; private_exponentiate_representative_with_scratch( modulus_q, @@ -3574,7 +3586,7 @@ impl RsaPrivateKeyComponents { .as_mut_slice() .get_mut(..prime_q.len()) .ok_or(RsaPrivateOpError::InvalidScratch)?, - &mut scratch.exponent_q_scratch, + &mut scratch.exponent_scratch, )?; private_import_unsigned_be_mod_to_fixed( @@ -3589,7 +3601,7 @@ impl RsaPrivateKeyComponents { .as_mut_slice() .get_mut(..prime_p.len()) .ok_or(RsaPrivateOpError::InvalidScratch)?, - &mut scratch.exponent_p_scratch, + &mut scratch.exponent_scratch, )?; private_sub_mod_unsigned_be_to_fixed( scratch @@ -3704,6 +3716,21 @@ impl SecretBigEndianInteger { }) } + fn new_fixed_width(bytes: &[u8], width: usize) -> Result { + if is_zero_unsigned_be(bytes) || bytes.len() > width { + return Err(RsaKeyError::InvalidModulus); + } + let mut fixed = vec![0u8; width]; + let offset = width.strict_sub(bytes.len()); + let Some(dst) = fixed.get_mut(offset..) else { + return Err(RsaKeyError::InvalidModulus); + }; + dst.copy_from_slice(bytes); + Ok(Self { + bytes: fixed.into_boxed_slice(), + }) + } + #[inline] fn as_bytes(&self) -> &[u8] { &self.bytes @@ -3899,38 +3926,51 @@ struct RsaPrivateExponentScratch { selected: SecretLimbs, reduced: SecretLimbs, table: SecretLimbs, - limb_count: usize, + max_limb_count: usize, +} + +struct RsaPrivateExponentWorkspace<'a> { + t: &'a mut [u64], + representative: &'a mut [u64], + one: &'a mut [u64], + base: &'a mut [u64], + acc: &'a mut [u64], + squared: &'a mut [u64], + multiplied: &'a mut [u64], + selected: &'a mut [u64], + reduced: &'a mut [u64], + table: &'a mut [u64], } impl RsaPrivateExponentScratch { - fn new(limb_count: usize) -> Self { + fn new(max_limb_count: usize) -> Self { Self { - t: SecretLimbs::zeroed(limb_count.strict_mul(2).strict_add(2)), - representative: SecretLimbs::zeroed(limb_count), - one: SecretLimbs::zeroed(limb_count), - base: SecretLimbs::zeroed(limb_count), - acc: SecretLimbs::zeroed(limb_count), - squared: SecretLimbs::zeroed(limb_count), - multiplied: SecretLimbs::zeroed(limb_count), - selected: SecretLimbs::zeroed(limb_count), - reduced: SecretLimbs::zeroed(limb_count), - table: SecretLimbs::zeroed(limb_count.strict_mul(PRIVATE_FIXED_WINDOW_TABLE_ENTRIES)), - limb_count, + t: SecretLimbs::zeroed(max_limb_count.strict_mul(2).strict_add(2)), + representative: SecretLimbs::zeroed(max_limb_count), + one: SecretLimbs::zeroed(max_limb_count), + base: SecretLimbs::zeroed(max_limb_count), + acc: SecretLimbs::zeroed(max_limb_count), + squared: SecretLimbs::zeroed(max_limb_count), + multiplied: SecretLimbs::zeroed(max_limb_count), + selected: SecretLimbs::zeroed(max_limb_count), + reduced: SecretLimbs::zeroed(max_limb_count), + table: SecretLimbs::zeroed(max_limb_count.strict_mul(PRIVATE_FIXED_WINDOW_TABLE_ENTRIES)), + max_limb_count, } } fn ensure_limb_count(&self, limb_count: usize) -> Result<(), RsaPrivateOpError> { - if self.limb_count == limb_count - && self.t.as_slice().len() == limb_count.strict_mul(2).strict_add(2) - && self.representative.as_slice().len() == limb_count - && self.one.as_slice().len() == limb_count - && self.base.as_slice().len() == limb_count - && self.acc.as_slice().len() == limb_count - && self.squared.as_slice().len() == limb_count - && self.multiplied.as_slice().len() == limb_count - && self.selected.as_slice().len() == limb_count - && self.reduced.as_slice().len() == limb_count - && self.table.as_slice().len() == limb_count.strict_mul(PRIVATE_FIXED_WINDOW_TABLE_ENTRIES) + if limb_count <= self.max_limb_count + && self.t.as_slice().len() >= limb_count.strict_mul(2).strict_add(2) + && self.representative.as_slice().len() >= limb_count + && self.one.as_slice().len() >= limb_count + && self.base.as_slice().len() >= limb_count + && self.acc.as_slice().len() >= limb_count + && self.squared.as_slice().len() >= limb_count + && self.multiplied.as_slice().len() >= limb_count + && self.selected.as_slice().len() >= limb_count + && self.reduced.as_slice().len() >= limb_count + && self.table.as_slice().len() >= limb_count.strict_mul(PRIVATE_FIXED_WINDOW_TABLE_ENTRIES) { Ok(()) } else { @@ -3938,6 +3978,24 @@ impl RsaPrivateExponentScratch { } } + fn workspace(&mut self, limb_count: usize) -> Result, RsaPrivateOpError> { + self.ensure_limb_count(limb_count)?; + let t_len = limb_count.strict_mul(2).strict_add(2); + let table_len = limb_count.strict_mul(PRIVATE_FIXED_WINDOW_TABLE_ENTRIES); + Ok(RsaPrivateExponentWorkspace { + t: private_scratch_prefix(&mut self.t, t_len)?, + representative: private_scratch_prefix(&mut self.representative, limb_count)?, + one: private_scratch_prefix(&mut self.one, limb_count)?, + base: private_scratch_prefix(&mut self.base, limb_count)?, + acc: private_scratch_prefix(&mut self.acc, limb_count)?, + squared: private_scratch_prefix(&mut self.squared, limb_count)?, + multiplied: private_scratch_prefix(&mut self.multiplied, limb_count)?, + selected: private_scratch_prefix(&mut self.selected, limb_count)?, + reduced: private_scratch_prefix(&mut self.reduced, limb_count)?, + table: private_scratch_prefix(&mut self.table, table_len)?, + }) + } + fn clear(&mut self) { ct::zeroize_words(self.t.as_mut_slice()); ct::zeroize_words(self.representative.as_mut_slice()); @@ -3952,6 +4010,13 @@ impl RsaPrivateExponentScratch { } } +fn private_scratch_prefix(limbs: &mut SecretLimbs, len: usize) -> Result<&mut [u64], RsaPrivateOpError> { + limbs + .as_mut_slice() + .get_mut(..len) + .ok_or(RsaPrivateOpError::InvalidScratch) +} + /// X.509 RSA public-key algorithm constraints. /// /// `rsaEncryption` keys are unconstrained RSA public keys. `id-RSASSA-PSS` @@ -5559,7 +5624,7 @@ impl fmt::Debug for RsaPublicKey { /// Caller-owned scratch for RSA public operations. pub struct RsaPublicScratch { - r2: Box<[u64]>, + montgomery: Box<[u64]>, limbs: Box<[u64]>, bytes: Box<[u8]>, limb_count: usize, @@ -5567,7 +5632,8 @@ pub struct RsaPublicScratch { } struct RsaPublicArithmeticScratch<'a> { - r2: &'a [u64], + modulus: &'a mut [u64], + r2: &'a mut [u64], t: &'a mut [u64], x: &'a mut [u64], tmp: &'a mut [u64], @@ -5581,8 +5647,14 @@ impl RsaPublicScratch { pub fn new(key: &RsaPublicKey) -> Self { let limbs = key.modulus.limbs.len(); let bytes = key.modulus.bytes.len(); + let mut montgomery = Vec::with_capacity(limbs.strict_mul(2)); + montgomery.extend_from_slice(&key.modulus.limbs); + montgomery.resize(limbs.strict_mul(2), 0); + let mut montgomery = montgomery.into_boxed_slice(); + let (_, r2) = montgomery.split_at_mut(limbs); + public_montgomery_r2_into(r2, &key.modulus.limbs); Self { - r2: public_montgomery_r2(&key.modulus.limbs), + montgomery, limbs: vec![0u64; limbs.strict_mul(6).strict_add(2)].into_boxed_slice(), bytes: vec![0u8; bytes.strict_mul(3)].into_boxed_slice(), limb_count: limbs, @@ -5591,11 +5663,11 @@ impl RsaPublicScratch { } fn arithmetic_scratch(&mut self) -> RsaPublicArithmeticScratch<'_> { - split_limb_scratch(&mut self.limbs, self.limb_count, &self.r2) + split_limb_scratch(&mut self.limbs, self.limb_count, &mut self.montgomery) } fn split_all(&mut self) -> (RsaPublicArithmeticScratch<'_>, &mut [u8], &mut [u8], &mut [u8]) { - let arithmetic_scratch = split_limb_scratch(&mut self.limbs, self.limb_count, &self.r2); + let arithmetic_scratch = split_limb_scratch(&mut self.limbs, self.limb_count, &mut self.montgomery); let (encoded, db, db_mask) = split_byte_scratch(&mut self.bytes, self.byte_count); (arithmetic_scratch, encoded, db, db_mask) } @@ -5609,7 +5681,12 @@ impl fmt::Debug for RsaPublicScratch { } } -fn split_limb_scratch<'a>(limbs: &'a mut [u64], limb_count: usize, r2: &'a [u64]) -> RsaPublicArithmeticScratch<'a> { +fn split_limb_scratch<'a>( + limbs: &'a mut [u64], + limb_count: usize, + montgomery: &'a mut [u64], +) -> RsaPublicArithmeticScratch<'a> { + let (modulus, r2) = montgomery.split_at_mut(limb_count); let t_len = limb_count.strict_mul(2).strict_add(2); let (t, rest) = limbs.split_at_mut(t_len); let (x, rest) = rest.split_at_mut(limb_count); @@ -5617,6 +5694,7 @@ fn split_limb_scratch<'a>(limbs: &'a mut [u64], limb_count: usize, r2: &'a [u64] let (base, rest) = rest.split_at_mut(limb_count); let (acc, _) = rest.split_at_mut(limb_count); RsaPublicArithmeticScratch { + modulus, r2, t, x, @@ -5712,9 +5790,11 @@ impl RsaPublicModulus { exponent: RsaPublicExponent, input: &[u8], out: &mut [u8], - scratch: RsaPublicArithmeticScratch<'_>, + mut scratch: RsaPublicArithmeticScratch<'_>, ) -> Result<(), RsaPublicOpError> { + self.bind_public_arithmetic_scratch(&mut scratch)?; let RsaPublicArithmeticScratch { + modulus: _, r2, t, x, @@ -5842,6 +5922,21 @@ impl RsaPublicModulus { Ok(()) } + fn bind_public_arithmetic_scratch( + &self, + scratch: &mut RsaPublicArithmeticScratch<'_>, + ) -> Result<(), RsaPublicOpError> { + let limbs = self.limbs.len(); + if scratch.modulus.len() != limbs || scratch.r2.len() != limbs { + return Err(RsaPublicOpError::InvalidScratch); + } + if scratch.modulus != self.limbs.as_ref() { + public_montgomery_r2_into(scratch.r2, &self.limbs); + scratch.modulus.copy_from_slice(&self.limbs); + } + Ok(()) + } + #[cfg(feature = "diag")] fn public_operation_bitserial( &self, @@ -5885,14 +5980,17 @@ impl RsaPublicModulus { out: &mut [u8], scratch: &mut RsaPublicScratch, ) -> Result<(), RsaPublicOpError> { + let mut arithmetic_scratch = scratch.arithmetic_scratch(); + self.bind_public_arithmetic_scratch(&mut arithmetic_scratch)?; let RsaPublicArithmeticScratch { + modulus: _, r2, t, x, tmp, base, acc, - } = scratch.arithmetic_scratch(); + } = arithmetic_scratch; let limbs = self.limbs.len(); if input.len() != self.bytes.len() || out.len() != self.bytes.len() { return Err(RsaPublicOpError::InvalidLength); @@ -5936,14 +6034,17 @@ impl RsaPublicModulus { out: &mut [u8], scratch: &mut RsaPublicScratch, ) -> Result<(), RsaPublicOpError> { + let mut arithmetic_scratch = scratch.arithmetic_scratch(); + self.bind_public_arithmetic_scratch(&mut arithmetic_scratch)?; let RsaPublicArithmeticScratch { + modulus: _, r2, t, x, tmp, base, acc, - } = scratch.arithmetic_scratch(); + } = arithmetic_scratch; let limbs = self.limbs.len(); if input.len() != self.bytes.len() || out.len() != self.bytes.len() { return Err(RsaPublicOpError::InvalidLength); @@ -6000,14 +6101,17 @@ impl RsaPublicModulus { out: &mut [u8], scratch: &mut RsaPublicScratch, ) -> Result<(), RsaPublicOpError> { + let mut arithmetic_scratch = scratch.arithmetic_scratch(); + self.bind_public_arithmetic_scratch(&mut arithmetic_scratch)?; let RsaPublicArithmeticScratch { + modulus: _, r2, t, x, tmp, base, acc, - } = scratch.arithmetic_scratch(); + } = arithmetic_scratch; let limbs = self.limbs.len(); if input.len() != self.bytes.len() || out.len() != self.bytes.len() { return Err(RsaPublicOpError::InvalidLength); @@ -6070,14 +6174,17 @@ impl RsaPublicModulus { out: &mut [u8], scratch: &mut RsaPublicScratch, ) -> Result<(), RsaPublicOpError> { + let mut arithmetic_scratch = scratch.arithmetic_scratch(); + self.bind_public_arithmetic_scratch(&mut arithmetic_scratch)?; let RsaPublicArithmeticScratch { + modulus: _, r2, t, x, tmp, base, acc, - } = scratch.arithmetic_scratch(); + } = arithmetic_scratch; let limbs = self.limbs.len(); if input.len() != self.bytes.len() || out.len() != self.bytes.len() { return Err(RsaPublicOpError::InvalidLength); @@ -6140,14 +6247,17 @@ impl RsaPublicModulus { out: &mut [u8], scratch: &mut RsaPublicScratch, ) -> Result<(), RsaPublicOpError> { + let mut arithmetic_scratch = scratch.arithmetic_scratch(); + self.bind_public_arithmetic_scratch(&mut arithmetic_scratch)?; let RsaPublicArithmeticScratch { + modulus: _, r2, t, x, tmp, base, acc, - } = scratch.arithmetic_scratch(); + } = arithmetic_scratch; let limbs = self.limbs.len(); if input.len() != self.bytes.len() || out.len() != self.bytes.len() { return Err(RsaPublicOpError::InvalidLength); @@ -6210,14 +6320,17 @@ impl RsaPublicModulus { out: &mut [u8], scratch: &mut RsaPublicScratch, ) -> Result<(), RsaPublicOpError> { + let mut arithmetic_scratch = scratch.arithmetic_scratch(); + self.bind_public_arithmetic_scratch(&mut arithmetic_scratch)?; let RsaPublicArithmeticScratch { + modulus: _, r2, t, x, tmp, base, acc, - } = scratch.arithmetic_scratch(); + } = arithmetic_scratch; let limbs = self.limbs.len(); if input.len() != self.bytes.len() || out.len() != self.bytes.len() { return Err(RsaPublicOpError::InvalidLength); @@ -6451,8 +6564,7 @@ fn parse_pkcs1_private_key_der_parts_with_policy<'a>( #[doc(hidden)] pub fn diag_rsa_validate_pkcs8_private_key_der(der: &[u8], policy: &RsaPublicKeyPolicy) -> Result { let components = parse_pkcs8_private_key_der_parts_with_policy(der, policy)?; - validate_modulus(components.modulus, policy)?; - validate_private_key_components(&components)?; + validate_private_key_import(&components, policy)?; Ok(components.modulus.len()) } @@ -6477,8 +6589,7 @@ pub fn diag_rsa_import_pkcs8_private_key_der_stage( stage: u8, ) -> Result { let components = parse_pkcs8_private_key_der_parts_with_policy(der, policy)?; - let modulus_bits = validate_modulus(components.modulus, policy)?; - validate_private_key_components(&components)?; + let modulus_bits = validate_private_key_import(&components, policy)?; if stage == 50 { return Ok(components.modulus.len()); } @@ -6507,8 +6618,8 @@ pub fn diag_rsa_import_pkcs8_private_key_der_stage( let private_exponent = SecretBigEndianInteger::new(components.private_exponent)?; let prime_p = SecretBigEndianInteger::new(components.prime_p)?; let prime_q = SecretBigEndianInteger::new(components.prime_q)?; - let exponent_p = SecretBigEndianInteger::new(components.exponent_p)?; - let exponent_q = SecretBigEndianInteger::new(components.exponent_q)?; + let exponent_p = SecretBigEndianInteger::new_fixed_width(components.exponent_p, prime_p_modulus.bytes.len())?; + let exponent_q = SecretBigEndianInteger::new_fixed_width(components.exponent_q, prime_q_modulus.bytes.len())?; let coefficient = SecretBigEndianInteger::new(components.coefficient)?; observed ^= private_exponent.bytes.len(); observed ^= prime_p.bytes.len(); @@ -6527,8 +6638,7 @@ fn private_key_components_from_parts( return Err(RsaKeyError::InvalidModulus); } - let modulus_bits = validate_modulus(components.modulus, policy)?; - validate_private_key_components(components)?; + let modulus_bits = validate_private_key_import(components, policy)?; let public = RsaPublicKey { modulus: RsaPublicModulus::new_with_montgomery_r2(components.modulus, modulus_bits), exponent: components.public_exponent, @@ -6543,8 +6653,8 @@ fn private_key_components_from_parts( prime_q: SecretBigEndianInteger::new(components.prime_q)?, prime_p_modulus, prime_q_modulus, - exponent_p: SecretBigEndianInteger::new(components.exponent_p)?, - exponent_q: SecretBigEndianInteger::new(components.exponent_q)?, + exponent_p: SecretBigEndianInteger::new_fixed_width(components.exponent_p, components.prime_p.len())?, + exponent_q: SecretBigEndianInteger::new_fixed_width(components.exponent_q, components.prime_q.len())?, coefficient: SecretBigEndianInteger::new(components.coefficient)?, }) } @@ -6600,6 +6710,26 @@ struct RsaPrivateKeyDerComponents<'a> { coefficient: &'a [u8], } +fn validate_private_key_import( + components: &RsaPrivateKeyDerComponents<'_>, + policy: &RsaPublicKeyPolicy, +) -> Result { + let modulus_bits = validate_modulus(components.modulus, policy)?; + validate_private_prime_factor(components.prime_p)?; + validate_private_prime_factor(components.prime_q)?; + + let factor_bits = modulus_bits.strict_add(1) / 2; + if unsigned_be_bit_len(components.prime_p) != factor_bits || unsigned_be_bit_len(components.prime_q) != factor_bits { + return Err(RsaKeyError::InvalidModulus); + } + + validate_private_key_components(components)?; + if !private_import_is_probable_prime(components.prime_p)? || !private_import_is_probable_prime(components.prime_q)? { + return Err(RsaKeyError::InvalidModulus); + } + Ok(modulus_bits) +} + fn read_zero_version(bytes: &[u8]) -> Result<(), RsaKeyError> { let version = read_positive_integer(bytes)?; if version == [0] { @@ -6650,50 +6780,66 @@ fn validate_private_key_components_through_stage( return Ok(()); } - let mut p_minus_one = vec![0u8; prime_p.len()]; - let mut q_minus_one = vec![0u8; prime_q.len()]; - private_import_decrement_unsigned_be_to_fixed(prime_p, &mut p_minus_one)?; - private_import_decrement_unsigned_be_to_fixed(prime_q, &mut q_minus_one)?; + let mut p_minus_one = SecretBigEndianBuffer::zeroed(prime_p.len()); + let mut q_minus_one = SecretBigEndianBuffer::zeroed(prime_q.len()); + private_import_decrement_unsigned_be_to_fixed(prime_p, p_minus_one.as_mut_slice())?; + private_import_decrement_unsigned_be_to_fixed(prime_q, q_minus_one.as_mut_slice())?; if stage == 30 { return Ok(()); } - let mut d_mod_p_minus_one = vec![0u8; p_minus_one.len()]; - private_import_unsigned_be_mod_to_len(private_exponent, &p_minus_one, &mut d_mod_p_minus_one)?; + let mut d_mod_p_minus_one = SecretBigEndianBuffer::zeroed(p_minus_one.as_slice().len()); + private_import_unsigned_be_mod_to_len( + private_exponent, + p_minus_one.as_slice(), + d_mod_p_minus_one.as_mut_slice(), + )?; if stage == 32 { return Ok(()); } - if !ct_eq_left_padded_unsigned_be(exponent_p, &d_mod_p_minus_one) { + if !ct_eq_left_padded_unsigned_be(exponent_p, d_mod_p_minus_one.as_slice()) { return Err(RsaKeyError::InvalidModulus); } if stage == 31 { return Ok(()); } - let mut d_mod_q_minus_one = vec![0u8; q_minus_one.len()]; - private_import_unsigned_be_mod_to_len(private_exponent, &q_minus_one, &mut d_mod_q_minus_one)?; - if !ct_eq_left_padded_unsigned_be(exponent_q, &d_mod_q_minus_one) { + let mut d_mod_q_minus_one = SecretBigEndianBuffer::zeroed(q_minus_one.as_slice().len()); + private_import_unsigned_be_mod_to_len( + private_exponent, + q_minus_one.as_slice(), + d_mod_q_minus_one.as_mut_slice(), + )?; + if !ct_eq_left_padded_unsigned_be(exponent_q, d_mod_q_minus_one.as_slice()) { return Err(RsaKeyError::InvalidModulus); } if stage == 3 { return Ok(()); } let public_exponent = public_exponent.as_u64().to_be_bytes(); - let mut e_times_d = vec![0u8; public_exponent.len().strict_add(private_exponent.len())]; - private_import_product_unsigned_be_to_fixed(&public_exponent, private_exponent, &mut e_times_d)?; + let mut e_times_d = SecretBigEndianBuffer::zeroed(public_exponent.len().strict_add(private_exponent.len())); + private_import_product_unsigned_be_to_fixed(&public_exponent, private_exponent, e_times_d.as_mut_slice())?; if stage == 40 { return Ok(()); } - let mut e_times_d_mod_p_minus_one = vec![0u8; p_minus_one.len()]; - private_import_unsigned_be_mod_to_len(&e_times_d, &p_minus_one, &mut e_times_d_mod_p_minus_one)?; - if !ct_eq_left_padded_unsigned_be(&[1], &e_times_d_mod_p_minus_one) { + let mut e_times_d_mod_p_minus_one = SecretBigEndianBuffer::zeroed(p_minus_one.as_slice().len()); + private_import_unsigned_be_mod_to_len( + e_times_d.as_slice(), + p_minus_one.as_slice(), + e_times_d_mod_p_minus_one.as_mut_slice(), + )?; + if !ct_eq_left_padded_unsigned_be(&[1], e_times_d_mod_p_minus_one.as_slice()) { return Err(RsaKeyError::InvalidModulus); } if stage == 41 { return Ok(()); } - let mut e_times_d_mod_q_minus_one = vec![0u8; q_minus_one.len()]; - private_import_unsigned_be_mod_to_len(&e_times_d, &q_minus_one, &mut e_times_d_mod_q_minus_one)?; - if !ct_eq_left_padded_unsigned_be(&[1], &e_times_d_mod_q_minus_one) { + let mut e_times_d_mod_q_minus_one = SecretBigEndianBuffer::zeroed(q_minus_one.as_slice().len()); + private_import_unsigned_be_mod_to_len( + e_times_d.as_slice(), + q_minus_one.as_slice(), + e_times_d_mod_q_minus_one.as_mut_slice(), + )?; + if !ct_eq_left_padded_unsigned_be(&[1], e_times_d_mod_q_minus_one.as_slice()) { return Err(RsaKeyError::InvalidModulus); } if stage == 42 { @@ -6703,24 +6849,18 @@ fn validate_private_key_components_through_stage( return Ok(()); } - let mut q_times_coefficient = vec![0u8; prime_q.len().strict_add(coefficient.len())]; - private_import_product_unsigned_be_to_fixed(prime_q, coefficient, &mut q_times_coefficient)?; - let mut q_times_coefficient_mod_p = vec![0u8; prime_p.len()]; - private_import_unsigned_be_mod_to_len(&q_times_coefficient, prime_p, &mut q_times_coefficient_mod_p)?; - if !ct_eq_left_padded_unsigned_be(&[1], &q_times_coefficient_mod_p) { + let mut q_times_coefficient = SecretBigEndianBuffer::zeroed(prime_q.len().strict_add(coefficient.len())); + private_import_product_unsigned_be_to_fixed(prime_q, coefficient, q_times_coefficient.as_mut_slice())?; + let mut q_times_coefficient_mod_p = SecretBigEndianBuffer::zeroed(prime_p.len()); + private_import_unsigned_be_mod_to_len( + q_times_coefficient.as_slice(), + prime_p, + q_times_coefficient_mod_p.as_mut_slice(), + )?; + if !ct_eq_left_padded_unsigned_be(&[1], q_times_coefficient_mod_p.as_slice()) { return Err(RsaKeyError::InvalidModulus); } - ct::zeroize(&mut p_minus_one); - ct::zeroize(&mut q_minus_one); - ct::zeroize(&mut d_mod_p_minus_one); - ct::zeroize(&mut d_mod_q_minus_one); - ct::zeroize(&mut e_times_d); - ct::zeroize(&mut e_times_d_mod_p_minus_one); - ct::zeroize(&mut e_times_d_mod_q_minus_one); - ct::zeroize(&mut q_times_coefficient); - ct::zeroize(&mut q_times_coefficient_mod_p); - Ok(()) } @@ -6734,6 +6874,56 @@ fn validate_private_prime_factor(bytes: &[u8]) -> Result<(), RsaKeyError> { Ok(()) } +fn private_import_is_probable_prime(candidate: &[u8]) -> Result { + if has_small_prime_factor(candidate) { + return Ok(false); + } + + let mut n_minus_one = SecretBigEndianBuffer::zeroed(candidate.len()); + private_import_decrement_unsigned_be_to_fixed(candidate, n_minus_one.as_mut_slice())?; + let mut odd_part = SecretBigEndianBuffer::zeroed(candidate.len()); + odd_part.as_mut_slice().copy_from_slice(n_minus_one.as_slice()); + let mut powers_of_two = 0usize; + while odd_part.as_slice().last().is_some_and(|byte| byte & 1 == 0) { + shift_right_one_unsigned_be(odd_part.as_mut_slice()); + powers_of_two = powers_of_two.strict_add(1); + } + + let modulus = private_component_modulus(candidate).map_err(|_| RsaKeyError::InvalidModulus)?; + for base_value in RSA_IMPORT_MILLER_RABIN_BASES { + let mut base = vec![0u8; candidate.len()]; + let base_bytes = base_value.to_be_bytes(); + let offset = base.len().strict_sub(base_bytes.len()); + let Some(base_suffix) = base.get_mut(offset..) else { + return Err(RsaKeyError::InvalidModulus); + }; + base_suffix.copy_from_slice(&base_bytes); + let accepted = miller_rabin_accepts_base( + &modulus, + odd_part.as_slice(), + powers_of_two, + n_minus_one.as_slice(), + &base, + ) + .map_err(|_| RsaKeyError::InvalidModulus)?; + ct::zeroize(&mut base); + if !accepted { + return Ok(false); + } + } + + Ok(true) +} + +fn shift_right_one_unsigned_be(bytes: &mut [u8]) { + let mut carry = 0u8; + for byte in bytes { + let next_carry = *byte & 1; + *byte = (*byte >> 1) | (carry << 7); + carry = next_carry; + } +} + fn validate_private_crt_component(component: &[u8], upper_bound: &[u8]) -> Result<(), RsaKeyError> { if !is_canonical_positive_unsigned_be(component) || !ct_unsigned_be_lt_public_shape(component, upper_bound) { return Err(RsaKeyError::InvalidModulus); @@ -6977,6 +7167,8 @@ fn keygen_build_private_key_from_primes( private_component_modulus(prime_p.as_slice()).map_err(|_| RsaKeyGenerationError::ArithmeticFailure)?; let prime_q_modulus = private_component_modulus(prime_q.as_slice()).map_err(|_| RsaKeyGenerationError::ArithmeticFailure)?; + let prime_p_bytes = prime_p_modulus.bytes.len(); + let prime_q_bytes = prime_q_modulus.bytes.len(); Ok(Some(RsaPrivateKeyComponents { public, @@ -6988,9 +7180,9 @@ fn keygen_build_private_key_from_primes( .map_err(|_| RsaKeyGenerationError::ArithmeticFailure)?, prime_p_modulus, prime_q_modulus, - exponent_p: SecretBigEndianInteger::from_vec(exponent_p.into_vec()) + exponent_p: SecretBigEndianInteger::new_fixed_width(exponent_p.as_slice(), prime_p_bytes) .map_err(|_| RsaKeyGenerationError::ArithmeticFailure)?, - exponent_q: SecretBigEndianInteger::from_vec(exponent_q.into_vec()) + exponent_q: SecretBigEndianInteger::new_fixed_width(exponent_q.as_slice(), prime_q_bytes) .map_err(|_| RsaKeyGenerationError::ArithmeticFailure)?, coefficient: SecretBigEndianInteger::from_vec(coefficient.into_vec()) .map_err(|_| RsaKeyGenerationError::ArithmeticFailure)?, @@ -7022,7 +7214,7 @@ fn keygen_generate_prime( tested_candidates = tested_candidates.strict_add(1); // Trial division rejects obvious composites before the expensive B.3 Miller-Rabin work. - if keygen_has_small_prime_factor(&candidate) || keygen_conflicts_with_public_exponent(&candidate) { + if has_small_prime_factor(&candidate) || keygen_conflicts_with_public_exponent(&candidate) { let mut candidate = candidate; ct::zeroize(&mut candidate); continue; @@ -7117,9 +7309,8 @@ fn keygen_bit_is_set(bytes: &[u8], bit: usize) -> bool { bytes.get(byte_index).is_some_and(|byte| byte & (1u8 << (bit % 8)) != 0) } -#[cfg(feature = "getrandom")] -fn keygen_has_small_prime_factor(candidate: &[u8]) -> bool { - RSA_KEYGEN_SMALL_PRIMES +fn has_small_prime_factor(candidate: &[u8]) -> bool { + RSA_SMALL_PRIMES .iter() .any(|&prime| unsigned_be_mod_u64(candidate, u64::from(prime)) == 0) } @@ -7160,16 +7351,28 @@ fn keygen_is_probable_prime(drbg: &mut RsaKeygenDrbg, candidate: &[u8]) -> Resul let mut odd_part = n_minus_one.as_slice().to_vec(); let mut powers_of_two = 0usize; while odd_part.last().is_some_and(|byte| byte & 1 == 0) { - keygen_shift_right_one(&mut odd_part); + shift_right_one_unsigned_be(&mut odd_part); odd_part = keygen_canonical_vec(odd_part); powers_of_two = powers_of_two.strict_add(1); } + let odd_part = SecretBigEndianBuffer::new(odd_part); + let mut odd_part_fixed = SecretBigEndianBuffer::zeroed(candidate.len()); + if let Err(err) = keygen_left_pad(odd_part.as_slice(), odd_part_fixed.as_mut_slice()) { + ct::zeroize(&mut n_minus_one_fixed); + return Err(err); + } let result = (|| { let modulus = private_component_modulus(candidate).map_err(|_| RsaKeyGenerationError::ArithmeticFailure)?; for _ in 0..keygen_miller_rabin_rounds(unsigned_be_bit_len(candidate)) { let mut base = keygen_random_miller_rabin_base(drbg, candidate, &n_minus_one_fixed)?; - let accepted = keygen_miller_rabin_accepts_base(&modulus, &odd_part, powers_of_two, &n_minus_one_fixed, &base); + let accepted = keygen_miller_rabin_accepts_base( + &modulus, + odd_part_fixed.as_slice(), + powers_of_two, + &n_minus_one_fixed, + &base, + ); ct::zeroize(&mut base); if !accepted? { return Ok(false); @@ -7179,7 +7382,6 @@ fn keygen_is_probable_prime(drbg: &mut RsaKeygenDrbg, candidate: &[u8]) -> Resul })(); ct::zeroize(&mut n_minus_one_fixed); - ct::zeroize(&mut odd_part); result } @@ -7223,22 +7425,33 @@ fn keygen_miller_rabin_accepts_base( n_minus_one_fixed: &[u8], base: &[u8], ) -> Result { + miller_rabin_accepts_base(modulus, odd_part, powers_of_two, n_minus_one_fixed, base) + .map_err(|_| RsaKeyGenerationError::ArithmeticFailure) +} + +fn miller_rabin_accepts_base( + modulus: &RsaPublicModulus, + odd_part: &[u8], + powers_of_two: usize, + n_minus_one_fixed: &[u8], + base: &[u8], +) -> Result { let mut x = vec![0u8; modulus.bytes.len()]; - if private_exponentiate_representative(modulus, odd_part, base, &mut x).is_err() { + if let Err(err) = private_exponentiate_representative(modulus, odd_part, base, &mut x) { ct::zeroize(&mut x); - return Err(RsaKeyGenerationError::ArithmeticFailure); + return Err(err); } - let mut accepted = keygen_is_one_fixed(&x) || ct::public_len_eq(&x, n_minus_one_fixed).declassify(); + let mut accepted = is_one_fixed(&x) || ct::public_len_eq(&x, n_minus_one_fixed).declassify(); for _ in 1..powers_of_two { if accepted { break; } let mut squared = vec![0u8; modulus.bytes.len()]; - if mod_mul_representatives(modulus, &x, &x, &mut squared).is_err() { + if let Err(err) = mod_mul_representatives(modulus, &x, &x, &mut squared) { ct::zeroize(&mut squared); ct::zeroize(&mut x); - return Err(RsaKeyGenerationError::ArithmeticFailure); + return Err(err); } x.copy_from_slice(&squared); ct::zeroize(&mut squared); @@ -7259,10 +7472,9 @@ fn keygen_prime_mod_inverse(value: &[u8], prime_modulus: &[u8]) -> Result> 1) | (carry << 7); - carry = next_carry; - } -} - #[cfg(feature = "getrandom")] fn keygen_sub_one_fixed(bytes: &mut [u8]) -> Result<(), RsaKeyGenerationError> { for byte in bytes.iter_mut().rev() { @@ -7444,8 +7646,7 @@ fn keygen_sub_one_fixed(bytes: &mut [u8]) -> Result<(), RsaKeyGenerationError> { Err(RsaKeyGenerationError::ArithmeticFailure) } -#[cfg(feature = "getrandom")] -fn keygen_is_one_fixed(bytes: &[u8]) -> bool { +fn is_one_fixed(bytes: &[u8]) -> bool { bytes.last() == Some(&1) && bytes .get(..bytes.len().saturating_sub(1)) @@ -7472,7 +7673,6 @@ fn keygen_canonical_vec(bytes: Vec) -> Vec { canonical.as_slice().to_vec() } -#[cfg(feature = "getrandom")] fn unsigned_be_bit_len(bytes: &[u8]) -> usize { let Some(first_nonzero) = bytes.iter().position(|&byte| byte != 0) else { return 0; @@ -7488,7 +7688,6 @@ fn unsigned_be_bit_len(bytes: &[u8]) -> usize { .strict_add(8usize.strict_sub(first.leading_zeros() as usize)) } -#[cfg(feature = "getrandom")] fn unsigned_be_mod_u64(bytes: &[u8], modulus: u64) -> u64 { let mut remainder = 0u128; let modulus = u128::from(modulus); @@ -8639,7 +8838,6 @@ fn private_sub_unsigned_be_to_len( } #[allow(clippy::indexing_slicing)] -#[cfg(feature = "getrandom")] fn private_exponentiate_representative( modulus: &RsaPublicModulus, exponent: &[u8], @@ -8648,7 +8846,7 @@ fn private_exponentiate_representative( ) -> Result<(), RsaPrivateOpError> { let bytes = modulus.bytes.len(); let limbs = modulus.limbs.len(); - if input.len() != bytes || out.len() != bytes || exponent.len() > bytes { + if input.len() != bytes || out.len() != bytes || exponent.len() != bytes { return Err(RsaPrivateOpError::InvalidLength); } @@ -8684,13 +8882,9 @@ fn private_exponentiate_representative( ); let mut table = private_fixed_window_table(base.as_slice(), acc.as_slice(), modulus, t.as_mut_slice()); - let leading_zero_bytes = bytes.strict_sub(exponent.len()); - for index in 0..bytes { - let exponent_byte = if index < leading_zero_bytes { - 0 - } else { - exponent[index.strict_sub(leading_zero_bytes)] - }; + let mut index = 0; + while index < bytes { + let exponent_byte = exponent[index]; private_exponentiate_window( &table, exponent_byte >> 4, @@ -8711,6 +8905,7 @@ fn private_exponentiate_representative( multiplied.as_mut_slice(), selected.as_mut_slice(), ); + index += 1; } ct::zeroize_words(table.as_mut_slice()); @@ -8729,81 +8924,65 @@ fn private_exponentiate_representative_with_scratch( ) -> Result<(), RsaPrivateOpError> { let bytes = modulus.bytes.len(); let limbs = modulus.limbs.len(); - if input.len() != bytes || out.len() != bytes || exponent.len() > bytes { + if input.len() != bytes || out.len() != bytes || exponent.len() != bytes { return Err(RsaPrivateOpError::InvalidLength); } - scratch.ensure_limb_count(limbs)?; - - limbs_from_be_into(input, scratch.representative.as_mut_slice()); - if !ct_limbs_lt(scratch.representative.as_slice(), &modulus.limbs) { + let RsaPrivateExponentWorkspace { + t, + representative, + one, + base, + acc, + squared, + multiplied, + selected, + reduced, + table, + } = scratch.workspace(limbs)?; + + limbs_from_be_into(input, representative); + if !ct_limbs_lt(representative, &modulus.limbs) { return Err(RsaPrivateOpError::RepresentativeOutOfRange); } - scratch.one.as_mut_slice().fill(0); - scratch.one.as_mut_slice()[0] = 1; - private_mont_mul( - scratch.base.as_mut_slice(), - scratch.representative.as_slice(), - modulus.montgomery_r2(), - modulus, - scratch.t.as_mut_slice(), - ); - private_mont_mul( - scratch.acc.as_mut_slice(), - scratch.one.as_slice(), - modulus.montgomery_r2(), - modulus, - scratch.t.as_mut_slice(), - ); + one.fill(0); + one[0] = 1; + private_mont_mul(base, representative, modulus.montgomery_r2(), modulus, t); + private_mont_mul(acc, one, modulus.montgomery_r2(), modulus, t); - private_fixed_window_table_into( - scratch.table.as_mut_slice(), - scratch.base.as_slice(), - scratch.acc.as_slice(), - modulus, - scratch.t.as_mut_slice(), - ); - let leading_zero_bytes = bytes.strict_sub(exponent.len()); - for index in 0..bytes { - let exponent_byte = if index < leading_zero_bytes { - 0 - } else { - exponent[index.strict_sub(leading_zero_bytes)] - }; + private_fixed_window_table_into(table, base, acc, modulus, t); + let mut index = 0; + while index < bytes { + let exponent_byte = exponent[index]; private_exponentiate_window( - scratch.table.as_slice(), + table, exponent_byte >> 4, modulus, - scratch.t.as_mut_slice(), - scratch.acc.as_mut_slice(), - scratch.squared.as_mut_slice(), - scratch.multiplied.as_mut_slice(), - scratch.selected.as_mut_slice(), + t, + acc, + squared, + multiplied, + selected, ); private_exponentiate_window( - scratch.table.as_slice(), + table, exponent_byte & 0x0f, modulus, - scratch.t.as_mut_slice(), - scratch.acc.as_mut_slice(), - scratch.squared.as_mut_slice(), - scratch.multiplied.as_mut_slice(), - scratch.selected.as_mut_slice(), + t, + acc, + squared, + multiplied, + selected, ); + index += 1; } - private_mont_reduce( - scratch.reduced.as_mut_slice(), - scratch.acc.as_slice(), - modulus, - scratch.t.as_mut_slice(), - ); - limbs_to_be(scratch.reduced.as_slice(), out); + private_mont_reduce(reduced, acc, modulus, t); + limbs_to_be(reduced, out); Ok(()) } #[allow(clippy::indexing_slicing)] -#[cfg(feature = "getrandom")] fn private_fixed_window_table( base: &[u64], one_montgomery: &[u64], @@ -8905,7 +9084,6 @@ fn private_choice_eq_mask_u8(left: u8, right: u8) -> u64 { 0u64.wrapping_sub(is_zero) } -#[cfg(feature = "getrandom")] fn mod_mul_representatives( modulus: &RsaPublicModulus, left: &[u8], @@ -9327,9 +9505,9 @@ fn private_import_unsigned_be_mod_to_fixed( if out.len() != modulus.bytes.len() { return Err(RsaPrivateOpError::InvalidLength); } - scratch.ensure_limb_count(modulus.limbs.len())?; + let workspace = scratch.workspace(modulus.limbs.len())?; - let remainder = scratch.representative.as_mut_slice(); + let remainder = workspace.representative; remainder.fill(0); for &byte in value { for bit in (0..8).rev() { @@ -9412,10 +9590,16 @@ fn limb_bit_len(limbs: &[u64]) -> usize { fn public_montgomery_r2(limbs: &[u64]) -> Box<[u64]> { let mut r2 = vec![0u64; limbs.len()]; - pow2_mod_into(&mut r2, limbs.len().strict_mul(128), limbs); + public_montgomery_r2_into(&mut r2, limbs); r2.into_boxed_slice() } +fn public_montgomery_r2_into(r2: &mut [u64], limbs: &[u64]) { + debug_assert_eq!(r2.len(), limbs.len()); + r2.fill(0); + pow2_mod_into(r2, limbs.len().strict_mul(128), limbs); +} + fn private_montgomery_r2(modulus: &[u8]) -> Result, RsaKeyError> { let limb_count = modulus.len().strict_add(7) / 8; let mut power = vec![0u8; limb_count.strict_mul(16).strict_add(1)]; @@ -10279,6 +10463,7 @@ mod tests { use alloc::format; use proptest::prelude::*; + use rsa::traits::{PrivateKeyParts as _, PublicKeyParts as _}; #[cfg(feature = "getrandom")] use serde_json::Value; @@ -10772,8 +10957,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 one: SecretBigEndianBuffer::zeroed(wrong_len), public_scratch: key.public_key().public_scratch(), mul_scratch: RsaPrivateMulScratch::new(key.components.public.modulus.limbs.len()), - exponent_p_scratch: RsaPrivateExponentScratch::new(key.components.prime_p_modulus.limbs.len()), - exponent_q_scratch: RsaPrivateExponentScratch::new(key.components.prime_q_modulus.limbs.len()), + exponent_scratch: RsaPrivateExponentScratch::new(key.components.public.modulus.limbs.len()), } } @@ -10787,11 +10971,184 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert_eq!(key.private_exponent.as_bytes(), rsa_private_exponent()); assert_eq!(key.prime_p.as_bytes(), rsa_private_prime_p()); assert_eq!(key.prime_q.as_bytes(), rsa_private_prime_q()); - assert_eq!(key.exponent_p.as_bytes(), rsa_private_exponent_p()); - assert_eq!(key.exponent_q.as_bytes(), rsa_private_exponent_q()); + let exponent_p = rsa_private_exponent_p(); + let exponent_q = rsa_private_exponent_q(); + assert_eq!(key.exponent_p.as_bytes().len(), key.prime_p.as_bytes().len()); + assert_eq!(key.exponent_q.as_bytes().len(), key.prime_q.as_bytes().len()); + assert!(key.exponent_p.as_bytes().ends_with(&exponent_p)); + assert!(key.exponent_q.as_bytes().ends_with(&exponent_q)); + assert!( + key.exponent_p.as_bytes()[..key.exponent_p.as_bytes().len().strict_sub(exponent_p.len())] + .iter() + .all(|&byte| byte == 0) + ); + assert!( + key.exponent_q.as_bytes()[..key.exponent_q.as_bytes().len().strict_sub(exponent_q.len())] + .iter() + .all(|&byte| byte == 0) + ); assert_eq!(key.coefficient.as_bytes(), rsa_private_coefficient()); } + #[test] + fn private_key_import_normalizes_crt_exponents_to_factor_width() { + let prime_p = rsa::BigUint::from_bytes_be(&rsa_private_prime_p()); + let prime_q = rsa::BigUint::from_bytes_be(&rsa_private_prime_q()); + let public_exponent = 883; + let oracle = rsa::RsaPrivateKey::from_p_q(prime_p, prime_q, rsa::BigUint::from(public_exponent)) + .expect("fixture factors and exponent form an algebraically valid key"); + let modulus = oracle.n().to_bytes_be(); + let private_exponent = oracle.d().to_bytes_be(); + let prime_p = oracle.primes()[0].to_bytes_be(); + let prime_q = oracle.primes()[1].to_bytes_be(); + let exponent_p = oracle.dp().expect("RustCrypto CRT exponent").to_bytes_be(); + let exponent_q = oracle.dq().expect("RustCrypto CRT exponent").to_bytes_be(); + assert!(exponent_p.len() < prime_p.len() || exponent_q.len() < prime_q.len()); + let coefficient = oracle + .qinv() + .and_then(|value| value.to_biguint()) + .expect("positive RustCrypto CRT coefficient") + .to_bytes_be(); + + let key = RsaPrivateKey::from_components_with_policy( + RsaPrivateKeyParts { + modulus: &modulus, + public_exponent, + private_exponent: &private_exponent, + prime_p: &prime_p, + prime_q: &prime_q, + exponent_p: &exponent_p, + exponent_q: &exponent_q, + coefficient: &coefficient, + }, + &RsaPublicKeyPolicy::legacy_verification().allow_legacy_odd_exponents(), + ) + .expect("algebraically valid components"); + + assert_eq!(key.components.exponent_p.as_bytes().len(), prime_p.len()); + assert_eq!(key.components.exponent_q.as_bytes().len(), prime_q.len()); + assert!(key.components.exponent_p.as_bytes().ends_with(&exponent_p)); + assert!(key.components.exponent_q.as_bytes().ends_with(&exponent_q)); + assert!( + key.components.exponent_p.as_bytes()[..prime_p.len().strict_sub(exponent_p.len())] + .iter() + .all(|&byte| byte == 0) + ); + assert!( + key.components.exponent_q.as_bytes()[..prime_q.len().strict_sub(exponent_q.len())] + .iter() + .all(|&byte| byte == 0) + ); + } + + #[test] + fn private_key_import_rejects_balanced_composite_factor_in_every_format() { + let one = rsa::BigUint::from(1u8); + let prime_q = rsa::BigUint::from_bytes_be(&rsa_private_prime_q()); + let (oracle, composite_p) = (1u64..4096) + .step_by(2) + .find_map(|offset| { + let left = (&one << 512usize) + rsa::BigUint::from(offset); + let right = (&one << 512usize) - rsa::BigUint::from(offset); + let composite_p = left * right; + let composite_bytes = composite_p.to_bytes_be(); + if unsigned_be_bit_len(&composite_bytes) != 1024 || has_small_prime_factor(&composite_bytes) { + return None; + } + rsa::RsaPrivateKey::from_p_q(composite_p.clone(), prime_q.clone(), rsa::BigUint::from(65_537u32)) + .ok() + .map(|oracle| (oracle, composite_p)) + }) + .expect("construct a balanced composite without a small prime divisor"); + let modulus = oracle.n().to_bytes_be(); + let private_exponent = oracle.d().to_bytes_be(); + let prime_p = oracle.primes()[0].to_bytes_be(); + let prime_q = oracle.primes()[1].to_bytes_be(); + let exponent_p = oracle.dp().expect("RustCrypto CRT exponent").to_bytes_be(); + let exponent_q = oracle.dq().expect("RustCrypto CRT exponent").to_bytes_be(); + let coefficient = oracle + .qinv() + .and_then(|value| value.to_biguint()) + .expect("positive RustCrypto CRT coefficient") + .to_bytes_be(); + assert_eq!(unsigned_be_bit_len(&modulus), 2048); + assert_eq!(unsigned_be_bit_len(&prime_p), 1024); + assert_eq!(unsigned_be_bit_len(&prime_q), 1024); + assert_eq!(prime_p, composite_p.to_bytes_be()); + assert!(!has_small_prime_factor(&prime_p)); + assert!(!private_import_is_probable_prime(&prime_p).unwrap()); + + assert_eq!( + RsaPrivateKey::from_components_with_policy( + RsaPrivateKeyParts { + modulus: &modulus, + public_exponent: 65_537, + private_exponent: &private_exponent, + prime_p: &prime_p, + prime_q: &prime_q, + exponent_p: &exponent_p, + exponent_q: &exponent_q, + coefficient: &coefficient, + }, + &RsaPublicKeyPolicy::legacy_verification(), + ) + .err(), + Some(RsaKeyError::InvalidModulus) + ); + + let pkcs1 = rsa::pkcs1::EncodeRsaPrivateKey::to_pkcs1_der(&oracle).expect("RustCrypto PKCS #1 encoding"); + assert_eq!( + RsaPrivateKey::from_pkcs1_der_with_policy(pkcs1.as_bytes(), &RsaPublicKeyPolicy::legacy_verification()).err(), + Some(RsaKeyError::InvalidModulus) + ); + let pkcs8 = rsa::pkcs8::EncodePrivateKey::to_pkcs8_der(&oracle).expect("RustCrypto PKCS #8 encoding"); + assert_eq!( + RsaPrivateKey::from_pkcs8_der_with_policy(pkcs8.as_bytes(), &RsaPublicKeyPolicy::legacy_verification()).err(), + Some(RsaKeyError::InvalidModulus) + ); + } + + #[test] + fn private_key_import_rejects_grossly_unbalanced_factors() { + let one = rsa::BigUint::from(1u8); + let oversized_composite = (&one << 3008usize) - &one; + let small_prime = rsa::BigUint::from(18_446_744_073_709_551_557u64); + let oracle = rsa::RsaPrivateKey::from_p_q(oversized_composite, small_prime, rsa::BigUint::from(65_537u32)) + .expect("algebraically valid two-factor key"); + let modulus = oracle.n().to_bytes_be(); + let private_exponent = oracle.d().to_bytes_be(); + let prime_p = oracle.primes()[0].to_bytes_be(); + let prime_q = oracle.primes()[1].to_bytes_be(); + let exponent_p = oracle.dp().expect("RustCrypto CRT exponent").to_bytes_be(); + let exponent_q = oracle.dq().expect("RustCrypto CRT exponent").to_bytes_be(); + let coefficient = oracle + .qinv() + .and_then(|value| value.to_biguint()) + .expect("positive RustCrypto CRT coefficient") + .to_bytes_be(); + assert_eq!(unsigned_be_bit_len(&modulus), 3072); + assert_eq!(unsigned_be_bit_len(&prime_p), 3008); + assert_eq!(unsigned_be_bit_len(&prime_q), 64); + + assert_eq!( + RsaPrivateKey::from_components_with_policy( + RsaPrivateKeyParts { + modulus: &modulus, + public_exponent: 65_537, + private_exponent: &private_exponent, + prime_p: &prime_p, + prime_q: &prime_q, + exponent_p: &exponent_p, + exponent_q: &exponent_q, + coefficient: &coefficient, + }, + &RsaPublicKeyPolicy::modern_verification(), + ) + .err(), + Some(RsaKeyError::InvalidModulus) + ); + } + #[test] fn private_key_components_debug_redacts_secret_material() { let der = test_pkcs1_private_key(); @@ -12298,7 +12655,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert!(keygen_probable_prime_meets_fips_lower_bound(&prime, 128)); assert_eq!(prime.last().copied().unwrap_or_default() & 1, 1); - assert!(!keygen_has_small_prime_factor(&prime)); + assert!(!has_small_prime_factor(&prime)); assert!(!keygen_conflicts_with_public_exponent(&prime)); assert!(keygen_is_probable_prime(&mut drbg, &prime).unwrap()); } @@ -12343,8 +12700,8 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[cfg(feature = "getrandom")] #[test] fn keygen_prefilter_rejects_small_prime_factors_without_rejecting_larger_prime() { - assert!(keygen_has_small_prime_factor(&999u16.to_be_bytes())); - assert!(!keygen_has_small_prime_factor(&1009u16.to_be_bytes())); + assert!(has_small_prime_factor(&999u16.to_be_bytes())); + assert!(!has_small_prime_factor(&1009u16.to_be_bytes())); } #[cfg(feature = "getrandom")] @@ -12375,14 +12732,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let prime = 1009u16.to_be_bytes(); let prime_modulus = private_component_modulus(&prime).unwrap(); assert!( - keygen_miller_rabin_accepts_base(&prime_modulus, &[63], 4, &1008u16.to_be_bytes(), &[0, 11]).unwrap(), + keygen_miller_rabin_accepts_base(&prime_modulus, &[0, 63], 4, &1008u16.to_be_bytes(), &[0, 11]).unwrap(), "1009 must pass a direct Miller-Rabin round for base 11" ); let composite = 341u16.to_be_bytes(); let composite_modulus = private_component_modulus(&composite).unwrap(); assert!( - !keygen_miller_rabin_accepts_base(&composite_modulus, &[85], 2, &340u16.to_be_bytes(), &[0, 2]).unwrap(), + !keygen_miller_rabin_accepts_base(&composite_modulus, &[0, 85], 2, &340u16.to_be_bytes(), &[0, 2]).unwrap(), "341 must fail a direct Miller-Rabin round for base 2" ); } @@ -12824,7 +13181,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert_eq!(unsigned_be_bit_len(&prime), 128); assert_eq!(prime.last().copied().unwrap_or_default() & 1, 1); - assert!(!keygen_has_small_prime_factor(&prime)); + assert!(!has_small_prime_factor(&prime)); assert!(!keygen_conflicts_with_public_exponent(&prime)); assert!(keygen_probable_prime_meets_fips_lower_bound(&prime, 128)); assert!(keygen_is_probable_prime(&mut drbg, &prime).unwrap()); @@ -12868,6 +13225,23 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 ); } + #[cfg(feature = "diag")] + #[test] + fn pkcs8_private_key_validation_stages_preserve_valid_result() { + let pkcs1 = test_pkcs1_private_key(); + let rsa_algorithm = algorithm_identifier(RSA_ENCRYPTION_OID, Some(&null())); + let der = test_pkcs8_private_key(&pkcs1, &rsa_algorithm); + let policy = RsaPublicKeyPolicy::legacy_verification(); + + for stage in [0, 1, 2, 30, 32, 31, 3, 40, 41, 42, 4, u8::MAX] { + assert_eq!( + diag_rsa_validate_pkcs8_private_key_der_stage(&der, &policy, stage), + Ok(rsa_private_modulus().len()), + "stage {stage}" + ); + } + } + #[test] fn private_key_parser_rejects_noncanonical_container_lengths_and_attributes() { let pkcs1 = test_pkcs1_private_key(); @@ -13365,12 +13739,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 } #[cfg(all( - target_arch = "aarch64", - target_os = "linux", + any( + all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos")), + all(target_arch = "x86_64", target_os = "linux") + ), not(feature = "portable-only"), not(miri) ))] - fn aarch64_linux_rsa_test_modulus(words: usize) -> RsaPublicModulus { + fn rsa_montgomery_test_modulus(words: usize) -> RsaPublicModulus { let mut bytes = vec![0u8; words.strict_mul(8)]; for (i, byte) in bytes.iter_mut().enumerate() { *byte = 0xa5u8 @@ -13383,12 +13759,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 } #[cfg(all( - target_arch = "aarch64", - target_os = "linux", + any( + all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos")), + all(target_arch = "x86_64", target_os = "linux") + ), not(feature = "portable-only"), not(miri) ))] - fn aarch64_linux_rsa_test_limbs(words: usize, seed: u64) -> Vec { + fn rsa_montgomery_test_limbs(words: usize, seed: u64) -> Vec { let mut state = seed ^ (words as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15); let mut limbs = vec![0u64; words]; for limb in &mut limbs { @@ -13403,8 +13781,10 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 } #[cfg(all( - target_arch = "aarch64", - target_os = "linux", + any( + all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos")), + all(target_arch = "x86_64", target_os = "linux") + ), not(feature = "portable-only"), not(miri) ))] @@ -13417,8 +13797,10 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 } #[cfg(all( - target_arch = "aarch64", - target_os = "linux", + any( + all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos")), + all(target_arch = "x86_64", target_os = "linux") + ), not(feature = "portable-only"), not(miri) ))] @@ -13443,6 +13825,74 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 mont_reduce_cios_portable(out, acc, modulus, t); } + #[cfg(all( + target_arch = "aarch64", + any(target_os = "linux", target_os = "macos"), + not(feature = "portable-only"), + not(miri) + ))] + macro_rules! assert_aarch64_rsa_montgomery_backend_matches_portable { + ($backend:ident) => { + for words in [32usize, 48, 64, 128] { + assert!($backend::supports_bignum_mont_words(words)); + let modulus = rsa_montgomery_test_modulus(words); + let a = rsa_montgomery_test_limbs(words, 0x243f_6a88_85a3_08d3); + let b = rsa_montgomery_test_limbs(words, 0x1319_8a2e_0370_7344); + let r2 = public_montgomery_r2(&modulus.limbs); + let scratch_words = $backend::bignum_mont_scratch_words(words); + let mut asm_t = vec![0u64; scratch_words]; + let mut portable_t = vec![0u64; words.strict_mul(2).strict_add(2)]; + let mut asm_out = vec![0u64; words]; + let mut portable_out = vec![0u64; words]; + + $backend::mont_mul_cios_words(&mut asm_out, &a, &b, &modulus.limbs, modulus.n0, words, &mut asm_t); + mont_mul_cios_portable(&mut portable_out, &a, &b, &modulus, &mut portable_t); + assert_limbs_match_as_bytes("mont_mul", words, &asm_out, &portable_out); + + asm_t.fill(0); + portable_t.fill(0); + let mut asm_square = a.clone(); + $backend::mont_square_cios_words_in_place(&mut asm_square, &modulus.limbs, modulus.n0, words, &mut asm_t); + mont_mul_cios_portable(&mut portable_out, &a, &a, &modulus, &mut portable_t); + assert_limbs_match_as_bytes("mont_square_in_place", words, &asm_square, &portable_out); + + asm_t.fill(0); + portable_t.fill(0); + let mut asm_left = a.clone(); + $backend::mont_mul_cios_words_in_place_left(&mut asm_left, &b, &modulus.limbs, modulus.n0, words, &mut asm_t); + mont_mul_cios_portable(&mut portable_out, &a, &b, &modulus, &mut portable_t); + assert_limbs_match_as_bytes("mont_mul_in_place_left", words, &asm_left, &portable_out); + + asm_t.fill(0); + portable_t.fill(0); + if words == 32 { + $backend::mont_reduce_cios_32(&mut asm_out, &a, &modulus.limbs, modulus.n0, &mut asm_t); + } else { + $backend::mont_reduce_cios_words(&mut asm_out, &a, &modulus.limbs, modulus.n0, words, &mut asm_t); + } + mont_reduce_cios_portable(&mut portable_out, &a, &modulus, &mut portable_t); + assert_limbs_match_as_bytes("mont_reduce", words, &asm_out, &portable_out); + + asm_t.fill(0); + portable_t.fill(0); + let mut asm_acc = vec![0u64; words]; + let mut portable_acc = vec![0u64; words]; + $backend::public_e65537_mont_words( + &mut asm_out, + &a, + &r2, + &mut asm_acc, + &modulus.limbs, + modulus.n0, + words, + &mut asm_t, + ); + public_e65537_cios_portable(&mut portable_out, &a, &r2, &mut portable_acc, &modulus, &mut portable_t); + assert_limbs_match_as_bytes("public_e65537", words, &asm_out, &portable_out); + } + }; + } + #[cfg(all( target_arch = "aarch64", target_os = "linux", @@ -13451,39 +13901,60 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 ))] #[test] fn aarch64_linux_rsa_montgomery_asm_matches_portable_across_supported_widths() { + assert_aarch64_rsa_montgomery_backend_matches_portable!(rsa_aarch64_linux_asm); + } + + #[cfg(all( + target_arch = "aarch64", + target_os = "macos", + not(feature = "portable-only"), + not(miri) + ))] + #[test] + fn aarch64_macos_rsa_montgomery_asm_matches_portable_across_supported_widths() { + assert_aarch64_rsa_montgomery_backend_matches_portable!(rsa_aarch64_asm); + } + + #[cfg(all( + target_arch = "x86_64", + target_os = "linux", + not(feature = "portable-only"), + not(miri) + ))] + #[test] + fn x86_64_linux_rsa_montgomery_asm_matches_portable_across_supported_widths() { for words in [32usize, 48, 64, 128] { - assert!(rsa_aarch64_linux_asm::supports_bignum_mont_words(words)); - let modulus = aarch64_linux_rsa_test_modulus(words); - let a = aarch64_linux_rsa_test_limbs(words, 0x243f_6a88_85a3_08d3); - let b = aarch64_linux_rsa_test_limbs(words, 0x1319_8a2e_0370_7344); + assert!( + rsa_x86_64_asm::supports_bignum_mont_words(words), + "RSA x86-64 assembly evidence requires BMI2 and ADX" + ); + assert!(rsa_x86_64_asm::supports_bignum_mont_square_words(words)); + + let modulus = rsa_montgomery_test_modulus(words); + let a = rsa_montgomery_test_limbs(words, 0x243f_6a88_85a3_08d3); + let b = rsa_montgomery_test_limbs(words, 0x1319_8a2e_0370_7344); let r2 = public_montgomery_r2(&modulus.limbs); - let scratch_words = rsa_aarch64_linux_asm::bignum_mont_scratch_words(words); + let scratch_words = rsa_x86_64_asm::bignum_mont_scratch_words(words); let mut asm_t = vec![0u64; scratch_words]; let mut portable_t = vec![0u64; words.strict_mul(2).strict_add(2)]; let mut asm_out = vec![0u64; words]; let mut portable_out = vec![0u64; words]; - rsa_aarch64_linux_asm::mont_mul_cios_words(&mut asm_out, &a, &b, &modulus.limbs, modulus.n0, words, &mut asm_t); + rsa_x86_64_asm::mont_mul_cios_words(&mut asm_out, &a, &b, &modulus.limbs, modulus.n0, words, &mut asm_t); mont_mul_cios_portable(&mut portable_out, &a, &b, &modulus, &mut portable_t); assert_limbs_match_as_bytes("mont_mul", words, &asm_out, &portable_out); asm_t.fill(0); portable_t.fill(0); let mut asm_square = a.clone(); - rsa_aarch64_linux_asm::mont_square_cios_words_in_place( - &mut asm_square, - &modulus.limbs, - modulus.n0, - words, - &mut asm_t, - ); + rsa_x86_64_asm::mont_square_cios_words_in_place(&mut asm_square, &modulus.limbs, modulus.n0, words, &mut asm_t); mont_mul_cios_portable(&mut portable_out, &a, &a, &modulus, &mut portable_t); assert_limbs_match_as_bytes("mont_square_in_place", words, &asm_square, &portable_out); asm_t.fill(0); portable_t.fill(0); let mut asm_left = a.clone(); - rsa_aarch64_linux_asm::mont_mul_cios_words_in_place_left( + rsa_x86_64_asm::mont_mul_cios_words_in_place_left( &mut asm_left, &b, &modulus.limbs, @@ -13496,28 +13967,22 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 asm_t.fill(0); portable_t.fill(0); - if words == 32 { - rsa_aarch64_linux_asm::mont_reduce_cios_32(&mut asm_out, &a, &modulus.limbs, modulus.n0, &mut asm_t); - } else { - rsa_aarch64_linux_asm::mont_reduce_cios_words(&mut asm_out, &a, &modulus.limbs, modulus.n0, words, &mut asm_t); + rsa_x86_64_asm::mont_mul_cios_words(&mut asm_out, &a, &r2, &modulus.limbs, modulus.n0, words, &mut asm_t); + let mut asm_acc = asm_out.clone(); + for _ in 0..16 { + rsa_x86_64_asm::mont_square_cios_words_in_place(&mut asm_acc, &modulus.limbs, modulus.n0, words, &mut asm_t); } - mont_reduce_cios_portable(&mut portable_out, &a, &modulus, &mut portable_t); - assert_limbs_match_as_bytes("mont_reduce", words, &asm_out, &portable_out); - - asm_t.fill(0); - portable_t.fill(0); - let mut asm_acc = vec![0u64; words]; - let mut portable_acc = vec![0u64; words]; - rsa_aarch64_linux_asm::public_e65537_mont_words( - &mut asm_out, - &a, - &r2, + rsa_x86_64_asm::mont_mul_cios_words_in_place_left( &mut asm_acc, + &asm_out, &modulus.limbs, modulus.n0, words, &mut asm_t, ); + mont_reduce_cios_portable(&mut asm_out, &asm_acc, &modulus, &mut asm_t); + + let mut portable_acc = vec![0u64; words]; public_e65537_cios_portable(&mut portable_out, &a, &r2, &mut portable_acc, &modulus, &mut portable_t); assert_limbs_match_as_bytes("public_e65537", words, &asm_out, &portable_out); } @@ -13620,10 +14085,11 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let spki = include_bytes!("../../benches/rsa_fixtures/rsa3072_spki.der"); let key = RsaPublicKey::from_spki_der(spki).unwrap(); let scratch = key.public_scratch(); + let (_, scratch_r2) = scratch.montgomery.split_at(scratch.limb_count); assert_eq!( diag_rsa_precompute_public_montgomery_r2(key.modulus()), - Ok(limb_checksum(&scratch.r2)) + Ok(limb_checksum(scratch_r2)) ); } diff --git a/src/auth/scrypt.rs b/src/auth/scrypt.rs index 864be5ed..96da4cd1 100644 --- a/src/auth/scrypt.rs +++ b/src/auth/scrypt.rs @@ -304,7 +304,6 @@ pub const fn required_caps(kernel: KernelId) -> crate::platform::Caps { /// Runtime dispatch for the active scrypt BlockMix backend. #[inline] -#[allow(dead_code)] // Reserved for Phase 4 dispatch; referenced by tests to pin the contract. fn active_kernel() -> KernelId { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] { diff --git a/src/backend/cache.rs b/src/backend/cache.rs index 5cc93c16..59027bcd 100644 --- a/src/backend/cache.rs +++ b/src/backend/cache.rs @@ -10,11 +10,108 @@ //! | no_std + atomics | Atomic state machine | Thread-safe, initialized once | //! | no_std - atomics | Direct call | Per-call computation | -#[cfg(all(not(feature = "std"), target_has_atomic = "ptr"))] +#[cfg(any(test, all(not(feature = "std"), target_has_atomic = "ptr")))] use core::cell::UnsafeCell; -#[cfg(all(not(feature = "std"), target_has_atomic = "ptr"))] +#[cfg(any(test, all(not(feature = "std"), target_has_atomic = "ptr")))] use core::mem::MaybeUninit; +#[cfg(any(test, all(not(feature = "std"), target_has_atomic = "ptr")))] +struct AtomicOnceCache { + state: core::sync::atomic::AtomicU8, + value: UnsafeCell>, +} + +#[cfg(any(test, all(not(feature = "std"), target_has_atomic = "ptr")))] +struct AtomicInitGuard<'a, T: Copy> { + cache: &'a AtomicOnceCache, + armed: bool, +} + +#[cfg(any(test, all(not(feature = "std"), target_has_atomic = "ptr")))] +impl AtomicInitGuard<'_, T> { + fn disarm(&mut self) { + self.armed = false; + } +} + +#[cfg(any(test, all(not(feature = "std"), target_has_atomic = "ptr")))] +impl Drop for AtomicInitGuard<'_, T> { + fn drop(&mut self) { + if self.armed { + self + .cache + .state + .store(AtomicOnceCache::::UNINIT, core::sync::atomic::Ordering::Release); + } + } +} + +#[cfg(any(test, all(not(feature = "std"), target_has_atomic = "ptr")))] +// SAFETY: `get_or_init` publishes the single initialized value with a Release +// store, and every reader observes that publication with an Acquire load. +unsafe impl Sync for AtomicOnceCache {} + +#[cfg(any(test, all(not(feature = "std"), target_has_atomic = "ptr")))] +impl AtomicOnceCache { + const UNINIT: u8 = 0; + const INITING: u8 = 1; + const READY: u8 = 2; + + const fn new() -> Self { + Self { + state: core::sync::atomic::AtomicU8::new(Self::UNINIT), + value: UnsafeCell::new(MaybeUninit::uninit()), + } + } + + fn get_or_init(&self, f: impl FnOnce() -> T) -> T { + use core::sync::atomic::Ordering; + + let mut initializer = Some(f); + + loop { + match self.state.load(Ordering::Acquire) { + Self::READY => { + // SAFETY: READY is published only after the value is initialized. The + // Acquire load observes the initializing thread's Release store. + return unsafe { (*self.value.get()).assume_init() }; + } + Self::UNINIT => { + if self + .state + .compare_exchange(Self::UNINIT, Self::INITING, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + continue; + } + + let mut guard = AtomicInitGuard { + cache: self, + armed: true, + }; + let value = initializer + .take() + .expect("initializer is consumed only by the CAS winner")(); + // SAFETY: The successful UNINIT-to-INITING transition gives this + // thread exclusive write access until it publishes READY. + unsafe { + (*self.value.get()).write(value); + } + self.state.store(Self::READY, Ordering::Release); + guard.disarm(); + return value; + } + Self::INITING => { + while self.state.load(Ordering::Acquire) == Self::INITING { + core::hint::spin_loop(); + } + } + _ => unreachable!("atomic cache state is private and has three values"), + } + } + } +} + /// A lazy cache for a single `Copy` value. /// /// Building block for dispatcher caching with proper synchronization. @@ -24,50 +121,13 @@ pub struct OnceCache { inner: std::sync::OnceLock, #[cfg(all(not(feature = "std"), target_has_atomic = "ptr"))] - state: core::sync::atomic::AtomicU8, - #[cfg(all(not(feature = "std"), target_has_atomic = "ptr"))] - value: UnsafeCell>, + inner: AtomicOnceCache, - // PhantomData<*const T> makes this !Send + !Sync on no-atomic targets. - // This is correct: no-atomic targets (e.g., thumbv6m) are single-threaded, - // so the linker will reject any attempt to use this across threads. #[cfg(all(not(feature = "std"), not(target_has_atomic = "ptr")))] - _marker: core::marker::PhantomData<*const T>, + _marker: core::marker::PhantomData, } -// The std path inherits OnceLock's Send/Sync bounds. The no_std atomic path -// stores through UnsafeCell, so Sync is implemented with the state-machine -// invariants below. - -#[cfg(all(not(feature = "std"), target_has_atomic = "ptr"))] -// SAFETY: OnceCache uses an atomic state machine (UNINIT -> INITING -> READY) -// to synchronize access to the UnsafeCell: -// - Only one thread can win the CAS from UNINIT to INITING -// - That thread has exclusive write access until it stores READY -// - All reads after READY are synchronized via Acquire/Release ordering -// - The state machine prevents data races on the inner value -#[allow(unsafe_code)] -unsafe impl Sync for OnceCache {} - -// no_std targets without atomics use PhantomData<*const T> to stay !Send + -// !Sync by default. Static dispatchers still need Sync, and these targets are -// single-threaded. -#[cfg(all(not(feature = "std"), not(target_has_atomic = "ptr")))] -// SAFETY: Targets without atomics (thumbv6m, etc.) are single-threaded by definition. -// There is no concurrent access possible, so Sync is trivially satisfied. -// The linker will reject any attempt to use threading primitives on these targets. -#[allow(unsafe_code)] -unsafe impl Sync for OnceCache {} - impl OnceCache { - /// State constants for the atomic state machine - #[cfg(all(not(feature = "std"), target_has_atomic = "ptr"))] - const UNINIT: u8 = 0; - #[cfg(all(not(feature = "std"), target_has_atomic = "ptr"))] - const INITING: u8 = 1; - #[cfg(all(not(feature = "std"), target_has_atomic = "ptr"))] - const READY: u8 = 2; - /// Create a new empty cache. #[must_use] pub const fn new() -> Self { @@ -76,9 +136,7 @@ impl OnceCache { inner: std::sync::OnceLock::new(), #[cfg(all(not(feature = "std"), target_has_atomic = "ptr"))] - state: core::sync::atomic::AtomicU8::new(0), - #[cfg(all(not(feature = "std"), target_has_atomic = "ptr"))] - value: UnsafeCell::new(MaybeUninit::uninit()), + inner: AtomicOnceCache::new(), #[cfg(all(not(feature = "std"), not(target_has_atomic = "ptr")))] _marker: core::marker::PhantomData, @@ -87,9 +145,9 @@ impl OnceCache { /// Get the cached value, initializing with `f` if not yet set. /// - /// On targets with atomics, this is thread-safe and the initializer - /// is called at most once. On targets without atomics, the initializer - /// is called on every invocation. + /// On targets with atomics, this is thread-safe and publishes one successful + /// initialization. A panicking initializer leaves the cache retryable. On + /// targets without atomics, the initializer is called on every invocation. /// /// Returns the cached value by copy (since T is Copy). #[inline] @@ -101,44 +159,7 @@ impl OnceCache { #[cfg(all(not(feature = "std"), target_has_atomic = "ptr"))] { - use core::sync::atomic::Ordering; - - let state = self.state.load(Ordering::Acquire); - if state == Self::READY { - // SAFETY: Value is fully initialized when state is READY. - // The Acquire load synchronizes with the Release store after initialization. - #[allow(unsafe_code)] - return unsafe { (*self.value.get()).assume_init() }; - } - - if state == Self::UNINIT - && self - .state - .compare_exchange(Self::UNINIT, Self::INITING, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - { - // This thread owns initialization until READY is published. - let value = f(); - // SAFETY: We hold exclusive access during INITING state. - // No other thread can observe or write to the value until we publish READY. - #[allow(unsafe_code)] - unsafe { - (*self.value.get()).write(value); - } - self.state.store(Self::READY, Ordering::Release); - return value; - } - - // Wait for the initializing thread to publish READY. - while self.state.load(Ordering::Acquire) != Self::READY { - core::hint::spin_loop(); - } - // SAFETY: State is READY, value is fully initialized. - // Acquire ordering ensures we see the write. - #[allow(unsafe_code)] - unsafe { - (*self.value.get()).assume_init() - } + self.inner.get_or_init(f) } #[cfg(all(not(feature = "std"), not(target_has_atomic = "ptr")))] @@ -202,6 +223,81 @@ mod tests { use super::*; + #[test] + fn test_atomic_once_cache_concurrent_init() { + static CALL_COUNT: AtomicUsize = AtomicUsize::new(0); + static CACHE: AtomicOnceCache = AtomicOnceCache::new(); + + let handles: Vec> = (0..10) + .map(|_| { + thread::spawn(|| { + for _ in 0..100 { + let value = CACHE.get_or_init(|| { + CALL_COUNT.fetch_add(1, Ordering::SeqCst); + 17 + }); + assert_eq!(value, 17); + } + }) + }) + .collect(); + + for handle in handles { + handle.join().unwrap(); + } + + assert_eq!(CALL_COUNT.load(Ordering::SeqCst), 1); + } + + #[test] + fn test_atomic_once_cache_recovers_after_initializer_panic() { + let cache = AtomicOnceCache::::new(); + + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + cache.get_or_init(|| panic!("controlled initializer panic")); + })); + + assert!(result.is_err()); + assert_eq!(cache.get_or_init(|| 23), 23); + } + + #[test] + fn test_atomic_once_cache_waiter_retries_after_initializer_panic() { + use std::{ + sync::{Arc, Barrier, mpsc}, + time::Duration, + }; + + let cache = Arc::new(AtomicOnceCache::::new()); + let initializer_entered = Arc::new(Barrier::new(2)); + let release_initializer = Arc::new(Barrier::new(2)); + + let panicking_cache = Arc::clone(&cache); + let panicking_entered = Arc::clone(&initializer_entered); + let panicking_release = Arc::clone(&release_initializer); + let panicking = thread::spawn(move || { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + panicking_cache.get_or_init(|| { + panicking_entered.wait(); + panicking_release.wait(); + panic!("controlled initializer panic"); + }); + })) + }); + + initializer_entered.wait(); + let waiting_cache = Arc::clone(&cache); + let (sender, receiver) = mpsc::channel(); + let waiting = thread::spawn(move || { + sender.send(waiting_cache.get_or_init(|| 29)).unwrap(); + }); + + release_initializer.wait(); + assert!(panicking.join().unwrap().is_err()); + assert_eq!(receiver.recv_timeout(Duration::from_secs(2)).unwrap(), 29); + waiting.join().unwrap(); + } + #[test] fn test_once_cache_concurrent_init() { static CALL_COUNT: AtomicUsize = AtomicUsize::new(0); diff --git a/src/checksum/common/mod.rs b/src/checksum/common/mod.rs index 4a051129..15032ae4 100644 --- a/src/checksum/common/mod.rs +++ b/src/checksum/common/mod.rs @@ -8,7 +8,7 @@ //! - Generic kernel selection and dispatch infrastructure //! - Generic test harnesses for CRC property testing //! - PCLMULQDQ/PMULL folding constants for hardware acceleration -//! - Software prefetch helpers for optimal memory access patterns +//! - Software prefetch helpers for large-buffer kernels // CLMUL folding constants and helpers (used by SIMD CRC backends). #[cfg(all( diff --git a/src/checksum/common/portable.rs b/src/checksum/common/portable.rs index d580cab1..cafbcbc9 100644 --- a/src/checksum/common/portable.rs +++ b/src/checksum/common/portable.rs @@ -11,20 +11,10 @@ //! Each table contains 256 entries representing the CRC contribution of a single //! byte at a specific position in the input stream. //! -//! The algorithm XORs the current CRC with N input bytes, then combines -//! N table lookups (one per byte position) using XOR. This achieves ~N× -//! throughput compared to byte-at-a-time processing. -//! -//! # Performance Characteristics -//! -//! | Width | Algorithm | Bytes/iter | Tables | Throughput | -//! |-------|-----------|------------|--------|------------| -//! | 16-bit | slice-by-4 | 4 | 4×256×u16 | ~1.5 GB/s | -//! | 16-bit | slice-by-8 | 8 | 8×256×u16 | ~2.5 GB/s | -//! | 32-bit | slice-by-8 | 8 | 8×256×u32 | ~4.0 GB/s | -//! | 32-bit | slice-by-16 | 16 | 16×256×u32 | ~5.0 GB/s | -//! | 64-bit | slice-by-8 | 8 | 8×256×u64 | ~2.0 GB/s | -//! | 64-bit | slice-by-16 | 16 | 16×256×u64 | ~3.0 GB/s | +//! The algorithm XORs the current CRC with N input bytes, then combines N +//! table lookups (one per byte position) using XOR. Each iteration consumes N +//! bytes at the cost of N tables; actual throughput depends on the target, +//! cache state, and input size. // SAFETY: All array indexing in this module uses bounded indices: // - chunks_exact guarantees chunk sizes @@ -423,7 +413,6 @@ pub fn slice8_64(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 8]) -> u64 { /// Update CRC-64 state using slice-by-16 algorithm. /// /// Processes 16 bytes per iteration (2× the CRC width in bytes). -/// Optimal for larger buffers where cache is warm. /// /// # Arguments /// @@ -478,7 +467,6 @@ fn slice16_64_scalar(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u6 /// Update CRC-64 state using slice-by-16 algorithm. /// /// Processes 16 bytes per iteration (2× the CRC width in bytes). -/// Optimal for larger buffers where cache is warm. /// /// # Arguments /// diff --git a/src/checksum/common/prefetch.rs b/src/checksum/common/prefetch.rs index 92dc9f72..ea4dba90 100644 --- a/src/checksum/common/prefetch.rs +++ b/src/checksum/common/prefetch.rs @@ -12,30 +12,15 @@ ), allow(dead_code, unused_imports) )] -//! This module provides platform-tuned prefetch constants and inline helpers -//! for optimal memory access patterns in large-buffer CRC computation. +//! This module provides architecture-specific prefetch distances and inline +//! helpers for large-buffer CRC computation. //! //! # Background //! -//! Modern CPUs have hardware prefetchers that work well for sequential access, -//! but software prefetch hints can still provide 5-15% gains in tight loops by: -//! - Reducing cache miss stalls when hardware prefetch falls behind -//! - Ensuring data arrives in L1 before the CPU needs it -//! - Working better with double-unrolled loops that process larger chunks -//! -//! # Prefetch Distance Tuning -//! -//! The optimal prefetch distance depends on: -//! - Memory latency (~70-100 cycles on modern x86, ~60-80 cycles on ARM) -//! - Loop iteration time (cycles per block processed) -//! - Cache line size (64 bytes on all modern platforms) -//! -//! Formula: `prefetch_distance = (memory_latency / cycles_per_block) * block_size` -//! -//! For a kernel processing 256B blocks at ~80 GiB/s on a 4GHz CPU: -//! - Time per block: 256B / 80GiB/s ≈ 3ns ≈ 12 cycles -//! - With 80-cycle memory latency: 80/12 * 256B ≈ 1.7KB -//! - Practical value: 512B-1KB (2-4 blocks ahead) +//! Software prefetch is only a hint. Whether it helps depends on the +//! microarchitecture, cache state, loop body, and input size. These distances +//! are manually maintained implementation choices, not portable performance +//! guarantees. //! //! # Usage Pattern //! @@ -61,26 +46,13 @@ /// Prefetch distance for large buffer kernels (xl size, 1MB+). /// -/// Tuned for double-unrolled loops processing 512B per iteration. -/// Value: 1024 bytes (2 iterations ahead). -/// -/// # Rationale -/// - At 80 GiB/s, 512B takes ~6ns ≈ 24 cycles at 4GHz -/// - Memory latency ~80-100 cycles on Zen4/Ice Lake -/// - 100 cycles / 24 cycles ≈ 4 blocks, but 2 blocks (1KB) is practical sweet spot -/// - Prefetching too far ahead wastes L1 cache space +/// The x86-64 folding loops use this 1,024-byte lookahead. #[cfg(target_arch = "x86_64")] pub const LARGE_BLOCK_DISTANCE: usize = 1024; /// Prefetch distance for large buffer kernels on ARM64. /// -/// Tuned for Graviton2/3 and Apple Silicon. -/// Value: 768 bytes (~2-3 iterations ahead for 256B blocks). -/// -/// # Rationale -/// - Graviton2: ~60-70 cycle memory latency, narrower memory bus than x86 -/// - Apple M1-M3: Excellent hardware prefetch, but software hints still help -/// - ARM NEON processes 128B blocks, so 768B = 6 blocks ahead +/// The AArch64 folding loops use this 768-byte lookahead. #[cfg(target_arch = "aarch64")] pub const LARGE_BLOCK_DISTANCE: usize = 768; diff --git a/src/checksum/crc16/aarch64.rs b/src/checksum/crc16/aarch64.rs index b640432e..64ce2ed1 100644 --- a/src/checksum/crc16/aarch64.rs +++ b/src/checksum/crc16/aarch64.rs @@ -1180,3 +1180,87 @@ pub fn crc16_ibm_pmull_eor3_3way_safe(crc: u16, data: &[u8]) -> u16 { ) } } + +#[cfg(test)] +mod tests { + extern crate std; + + use alloc::vec::Vec; + + use super::*; + + const LENS: &[usize] = &[0, 1, 7, 15, 16, 31, 63, 64, 127, 128, 255, 256, 1023, 1024, 4096]; + const OFFSETS: &[usize] = &[0, 1, 7, 15]; + const STATES: &[u16] = &[0, 0x1d0f, 0xa5a5, u16::MAX]; + + fn data() -> Vec { + (0..4111) + .map(|i| (i as u8).wrapping_mul(29).wrapping_add((i >> 8) as u8)) + .collect() + } + + fn assert_kernel(name: &str, kernel: fn(u16, &[u8]) -> u16, portable: fn(u16, &[u8]) -> u16) { + let input = data(); + for &state in STATES { + for &offset in OFFSETS { + for &len in LENS { + let slice = &input[offset..offset + len]; + assert_eq!( + kernel(state, slice), + portable(state, slice), + "{name} state={state:#06x} offset={offset} len={len}" + ); + } + } + } + } + + #[test] + fn pmull_kernels_match_portable() { + if !crate::platform::caps().has(crate::platform::caps::aarch64::PMULL_READY) { + return; + } + + for (name, kernel) in [ + ("ccitt/pmull", crc16_ccitt_pmull_safe as fn(u16, &[u8]) -> u16), + ("ccitt/pmull-small", crc16_ccitt_pmull_small_safe), + ("ccitt/pmull-2way", crc16_ccitt_pmull_2way_safe), + ("ccitt/pmull-3way", crc16_ccitt_pmull_3way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc16_ccitt_slice8); + } + + for (name, kernel) in [ + ("ibm/pmull", crc16_ibm_pmull_safe as fn(u16, &[u8]) -> u16), + ("ibm/pmull-small", crc16_ibm_pmull_small_safe), + ("ibm/pmull-2way", crc16_ibm_pmull_2way_safe), + ("ibm/pmull-3way", crc16_ibm_pmull_3way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc16_ibm_slice8); + } + } + + #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] + #[test] + fn pmull_eor3_kernels_match_portable() { + if !crate::platform::caps().has(crate::platform::caps::aarch64::PMULL_EOR3_READY) { + return; + } + + for (name, kernel) in [ + ("ccitt/pmull-eor3", crc16_ccitt_pmull_eor3_safe as fn(u16, &[u8]) -> u16), + ("ccitt/pmull-eor3-2way", crc16_ccitt_pmull_eor3_2way_safe), + ("ccitt/pmull-eor3-3way", crc16_ccitt_pmull_eor3_3way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc16_ccitt_slice8); + } + + for (name, kernel) in [ + ("ibm/pmull-eor3", crc16_ibm_pmull_eor3_safe as fn(u16, &[u8]) -> u16), + ("ibm/pmull-eor3-2way", crc16_ibm_pmull_eor3_2way_safe), + ("ibm/pmull-eor3-3way", crc16_ibm_pmull_eor3_3way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc16_ibm_slice8); + } + } +} diff --git a/src/checksum/crc16/config.rs b/src/checksum/crc16/config.rs index 80dad0e3..2649ddb1 100644 --- a/src/checksum/crc16/config.rs +++ b/src/checksum/crc16/config.rs @@ -1,7 +1,7 @@ //! CRC-16 runtime configuration. //! //! This module handles force mode selection for CRC-16. The dispatch module -//! handles optimal kernel selection automatically; this module only provides +//! handles kernel selection automatically; this module only provides //! the ability to force specific backends for testing/debugging. use crate::platform::Caps; diff --git a/src/checksum/crc16/portable.rs b/src/checksum/crc16/portable.rs index b41f6f31..650b3db5 100644 --- a/src/checksum/crc16/portable.rs +++ b/src/checksum/crc16/portable.rs @@ -19,8 +19,7 @@ pub fn crc16_ibm_slice8(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT byte-at-a-time lookup computation. /// -/// This is typically faster than slice-by-8 for tiny buffers because it uses a -/// single 256-entry table. +/// Uses one 256-entry table rather than the slice-by-8 table set. #[inline(always)] pub fn crc16_ccitt_bytewise(crc: u16, data: &[u8]) -> u16 { crc16_bytewise(crc, data, &kernel_tables::CCITT_TABLES_8[0]) @@ -28,8 +27,7 @@ pub fn crc16_ccitt_bytewise(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM byte-at-a-time lookup computation. /// -/// This is typically faster than slice-by-8 for tiny buffers because it uses a -/// single 256-entry table. +/// Uses one 256-entry table rather than the slice-by-8 table set. #[inline(always)] pub fn crc16_ibm_bytewise(crc: u16, data: &[u8]) -> u16 { crc16_bytewise(crc, data, &kernel_tables::IBM_TABLES_8[0]) diff --git a/src/checksum/crc16/power.rs b/src/checksum/crc16/power.rs index 7da92fc0..5e145fa5 100644 --- a/src/checksum/crc16/power.rs +++ b/src/checksum/crc16/power.rs @@ -880,3 +880,59 @@ pub fn crc16_ibm_vpmsum_8way_safe(crc: u16, data: &[u8]) -> u16 { ) } } + +#[cfg(test)] +mod tests { + extern crate std; + + use alloc::vec::Vec; + + use super::*; + + const LENS: &[usize] = &[0, 1, 7, 15, 16, 31, 63, 64, 127, 128, 255, 256, 1023, 1024, 4096]; + const OFFSETS: &[usize] = &[0, 1, 7, 15]; + const STATES: &[u16] = &[0, 0x1d0f, 0xa5a5, u16::MAX]; + + fn assert_kernel(name: &str, kernel: fn(u16, &[u8]) -> u16, portable: fn(u16, &[u8]) -> u16) { + let input: Vec = (0..4111) + .map(|i| (i as u8).wrapping_mul(17).wrapping_add((i >> 8) as u8)) + .collect(); + for &state in STATES { + for &offset in OFFSETS { + for &len in LENS { + let slice = &input[offset..offset + len]; + assert_eq!( + kernel(state, slice), + portable(state, slice), + "{name} state={state:#06x} offset={offset} len={len}" + ); + } + } + } + } + + #[test] + fn vpmsum_kernels_match_portable() { + if !crate::platform::caps().has(crate::platform::caps::power::VPMSUM_READY) { + return; + } + + for (name, kernel) in [ + ("ccitt/vpmsum", crc16_ccitt_vpmsum_safe as fn(u16, &[u8]) -> u16), + ("ccitt/vpmsum-2way", crc16_ccitt_vpmsum_2way_safe), + ("ccitt/vpmsum-4way", crc16_ccitt_vpmsum_4way_safe), + ("ccitt/vpmsum-8way", crc16_ccitt_vpmsum_8way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc16_ccitt_slice8); + } + + for (name, kernel) in [ + ("ibm/vpmsum", crc16_ibm_vpmsum_safe as fn(u16, &[u8]) -> u16), + ("ibm/vpmsum-2way", crc16_ibm_vpmsum_2way_safe), + ("ibm/vpmsum-4way", crc16_ibm_vpmsum_4way_safe), + ("ibm/vpmsum-8way", crc16_ibm_vpmsum_8way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc16_ibm_slice8); + } + } +} diff --git a/src/checksum/crc16/riscv64.rs b/src/checksum/crc16/riscv64.rs index 2f12c835..f042f18b 100644 --- a/src/checksum/crc16/riscv64.rs +++ b/src/checksum/crc16/riscv64.rs @@ -1090,3 +1090,82 @@ pub fn crc16_ibm_zvbc_4way_safe(crc: u16, data: &[u8]) -> u16 { ) } } + +#[cfg(test)] +mod tests { + extern crate std; + + use alloc::vec::Vec; + + use super::*; + + const LENS: &[usize] = &[0, 1, 7, 15, 16, 31, 63, 64, 127, 128, 255, 256, 1023, 1024, 4096]; + const OFFSETS: &[usize] = &[0, 1, 7, 15]; + const STATES: &[u16] = &[0, 0x1d0f, 0xa5a5, u16::MAX]; + + fn assert_kernel(name: &str, kernel: fn(u16, &[u8]) -> u16, portable: fn(u16, &[u8]) -> u16) { + let input: Vec = (0..4111) + .map(|i| (i as u8).wrapping_mul(13).wrapping_add((i >> 8) as u8)) + .collect(); + for &state in STATES { + for &offset in OFFSETS { + for &len in LENS { + let slice = &input[offset..offset + len]; + assert_eq!( + kernel(state, slice), + portable(state, slice), + "{name} state={state:#06x} offset={offset} len={len}" + ); + } + } + } + } + + #[test] + fn zbc_kernels_match_portable() { + if !crate::platform::caps().has(crate::platform::caps::riscv::ZBC) { + return; + } + + for (name, kernel) in [ + ("ccitt/zbc", crc16_ccitt_zbc_safe as fn(u16, &[u8]) -> u16), + ("ccitt/zbc-2way", crc16_ccitt_zbc_2way_safe), + ("ccitt/zbc-4way", crc16_ccitt_zbc_4way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc16_ccitt_slice8); + } + + for (name, kernel) in [ + ("ibm/zbc", crc16_ibm_zbc_safe as fn(u16, &[u8]) -> u16), + ("ibm/zbc-2way", crc16_ibm_zbc_2way_safe), + ("ibm/zbc-4way", crc16_ibm_zbc_4way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc16_ibm_slice8); + } + } + + #[test] + fn zvbc_kernels_match_portable() { + use crate::platform::caps::riscv; + + if !crate::platform::caps().has(riscv::V.union(riscv::ZVBC)) { + return; + } + + for (name, kernel) in [ + ("ccitt/zvbc", crc16_ccitt_zvbc_safe as fn(u16, &[u8]) -> u16), + ("ccitt/zvbc-2way", crc16_ccitt_zvbc_2way_safe), + ("ccitt/zvbc-4way", crc16_ccitt_zvbc_4way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc16_ccitt_slice8); + } + + for (name, kernel) in [ + ("ibm/zvbc", crc16_ibm_zvbc_safe as fn(u16, &[u8]) -> u16), + ("ibm/zvbc-2way", crc16_ibm_zvbc_2way_safe), + ("ibm/zvbc-4way", crc16_ibm_zvbc_4way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc16_ibm_slice8); + } + } +} diff --git a/src/checksum/crc16/s390x.rs b/src/checksum/crc16/s390x.rs index f5d2297d..fac679a1 100644 --- a/src/checksum/crc16/s390x.rs +++ b/src/checksum/crc16/s390x.rs @@ -558,3 +558,57 @@ pub fn crc16_ibm_vgfm_4way_safe(crc: u16, data: &[u8]) -> u16 { ) } } + +#[cfg(test)] +mod tests { + extern crate std; + + use alloc::vec::Vec; + + use super::*; + + const LENS: &[usize] = &[0, 1, 7, 15, 16, 31, 63, 64, 127, 128, 255, 256, 1023, 1024, 4096]; + const OFFSETS: &[usize] = &[0, 1, 7, 15]; + const STATES: &[u16] = &[0, 0x1d0f, 0xa5a5, u16::MAX]; + + fn assert_kernel(name: &str, kernel: fn(u16, &[u8]) -> u16, portable: fn(u16, &[u8]) -> u16) { + let input: Vec = (0..4111) + .map(|i| (i as u8).wrapping_mul(43).wrapping_add((i >> 8) as u8)) + .collect(); + for &state in STATES { + for &offset in OFFSETS { + for &len in LENS { + let slice = &input[offset..offset + len]; + assert_eq!( + kernel(state, slice), + portable(state, slice), + "{name} state={state:#06x} offset={offset} len={len}" + ); + } + } + } + } + + #[test] + fn vgfm_kernels_match_portable() { + if !crate::platform::caps().has(crate::platform::caps::s390x::VECTOR) { + return; + } + + for (name, kernel) in [ + ("ccitt/vgfm", crc16_ccitt_vgfm_safe as fn(u16, &[u8]) -> u16), + ("ccitt/vgfm-2way", crc16_ccitt_vgfm_2way_safe), + ("ccitt/vgfm-4way", crc16_ccitt_vgfm_4way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc16_ccitt_slice8); + } + + for (name, kernel) in [ + ("ibm/vgfm", crc16_ibm_vgfm_safe as fn(u16, &[u8]) -> u16), + ("ibm/vgfm-2way", crc16_ibm_vgfm_2way_safe), + ("ibm/vgfm-4way", crc16_ibm_vgfm_4way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc16_ibm_slice8); + } + } +} diff --git a/src/checksum/crc16/x86_64.rs b/src/checksum/crc16/x86_64.rs index d6d53451..95b270c7 100644 --- a/src/checksum/crc16/x86_64.rs +++ b/src/checksum/crc16/x86_64.rs @@ -370,7 +370,7 @@ unsafe fn update_simd_width32_reflected_4way( acc[6] = s0[6].fold_16_reflected(c384, acc[6]); acc[7] = s0[7].fold_16_reflected(c384, acc[7]); - for block in &blocks[aligned..] { + for block in &blocks[quad_aligned..] { fold_block_128_reflected(&mut acc, block, coeff_128); } @@ -1712,3 +1712,90 @@ pub fn crc16_ibm_vpclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { ) } } + +#[cfg(test)] +mod tests { + extern crate std; + + use alloc::vec::Vec; + + use super::*; + + const LENS: &[usize] = &[0, 1, 7, 15, 16, 31, 63, 64, 127, 128, 255, 256, 1023, 1024, 4096]; + const OFFSETS: &[usize] = &[0, 1, 7, 15]; + const STATES: &[u16] = &[0, 0x1d0f, 0xa5a5, u16::MAX]; + + fn assert_kernel(name: &str, kernel: fn(u16, &[u8]) -> u16, portable: fn(u16, &[u8]) -> u16) { + let input: Vec = (0..4111) + .map(|i| (i as u8).wrapping_mul(59).wrapping_add((i >> 8) as u8)) + .collect(); + for &state in STATES { + for &offset in OFFSETS { + for &len in LENS { + let slice = &input[offset..offset + len]; + assert_eq!( + kernel(state, slice), + portable(state, slice), + "{name} state={state:#06x} offset={offset} len={len}" + ); + } + } + } + } + + #[test] + fn pclmul_kernels_match_portable() { + if !crate::platform::caps().has(crate::platform::caps::x86::PCLMUL_READY) { + return; + } + + for (name, kernel) in [ + ("ccitt/pclmul", crc16_ccitt_pclmul_safe as fn(u16, &[u8]) -> u16), + ("ccitt/pclmul-small", crc16_ccitt_pclmul_small_safe), + ("ccitt/pclmul-2way", crc16_ccitt_pclmul_2way_safe), + ("ccitt/pclmul-4way", crc16_ccitt_pclmul_4way_safe), + ("ccitt/pclmul-7way", crc16_ccitt_pclmul_7way_safe), + ("ccitt/pclmul-8way", crc16_ccitt_pclmul_8way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc16_ccitt_slice8); + } + + for (name, kernel) in [ + ("ibm/pclmul", crc16_ibm_pclmul_safe as fn(u16, &[u8]) -> u16), + ("ibm/pclmul-small", crc16_ibm_pclmul_small_safe), + ("ibm/pclmul-2way", crc16_ibm_pclmul_2way_safe), + ("ibm/pclmul-4way", crc16_ibm_pclmul_4way_safe), + ("ibm/pclmul-7way", crc16_ibm_pclmul_7way_safe), + ("ibm/pclmul-8way", crc16_ibm_pclmul_8way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc16_ibm_slice8); + } + } + + #[test] + fn vpclmul_kernels_match_portable() { + if !crate::platform::caps().has(crate::platform::caps::x86::VPCLMUL_READY) { + return; + } + + for (name, kernel) in [ + ("ccitt/vpclmul", crc16_ccitt_vpclmul_safe as fn(u16, &[u8]) -> u16), + ("ccitt/vpclmul-2way", crc16_ccitt_vpclmul_2way_safe), + ("ccitt/vpclmul-4way", crc16_ccitt_vpclmul_4way_safe), + ("ccitt/vpclmul-7way", crc16_ccitt_vpclmul_7way_safe), + ("ccitt/vpclmul-8way", crc16_ccitt_vpclmul_8way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc16_ccitt_slice8); + } + + for (name, kernel) in [ + ("ibm/vpclmul", crc16_ibm_vpclmul_safe as fn(u16, &[u8]) -> u16), + ("ibm/vpclmul-2way", crc16_ibm_vpclmul_2way_safe), + ("ibm/vpclmul-4way", crc16_ibm_vpclmul_4way_safe), + ("ibm/vpclmul-7way", crc16_ibm_vpclmul_7way_safe), + ("ibm/vpclmul-8way", crc16_ibm_vpclmul_8way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc16_ibm_slice8); + } + } +} diff --git a/src/checksum/crc24/aarch64.rs b/src/checksum/crc24/aarch64.rs index f76a60ad..294e96a7 100644 --- a/src/checksum/crc24/aarch64.rs +++ b/src/checksum/crc24/aarch64.rs @@ -643,3 +643,50 @@ pub fn crc24_openpgp_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { }; from_reflected_state(state) } + +#[cfg(test)] +mod tests { + extern crate std; + + use alloc::vec::Vec; + + use super::*; + + const LENS: &[usize] = &[0, 1, 7, 15, 16, 31, 63, 64, 127, 128, 255, 256, 1023, 1024, 4096]; + const OFFSETS: &[usize] = &[0, 1, 7, 15]; + const STATES: &[u32] = &[0, 0x00b7_04ce, 0x005a_a5a5, 0x00ff_ffff]; + + fn data() -> Vec { + (0..4111) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add((i >> 8) as u8)) + .collect() + } + + #[test] + fn pmull_kernels_match_portable() { + if !crate::platform::caps().has(crate::platform::caps::aarch64::PMULL_READY) { + return; + } + + let input = data(); + for (name, kernel) in [ + ("openpgp/pmull", crc24_openpgp_pmull_safe as fn(u32, &[u8]) -> u32), + ("openpgp/pmull-small", crc24_openpgp_pmull_small_safe), + ("openpgp/pmull-2way", crc24_openpgp_pmull_2way_safe), + ("openpgp/pmull-3way", crc24_openpgp_pmull_3way_safe), + ] { + for &state in STATES { + for &offset in OFFSETS { + for &len in LENS { + let slice = &input[offset..offset + len]; + assert_eq!( + kernel(state, slice), + super::super::portable::crc24_openpgp_slice8(state, slice), + "{name} state={state:#010x} offset={offset} len={len}" + ); + } + } + } + } + } +} diff --git a/src/checksum/crc24/config.rs b/src/checksum/crc24/config.rs index 76aab81f..ac14fdbe 100644 --- a/src/checksum/crc24/config.rs +++ b/src/checksum/crc24/config.rs @@ -1,7 +1,7 @@ //! CRC-24 runtime configuration. //! //! This module handles force mode selection for CRC-24. The dispatch module -//! handles optimal kernel selection automatically; this module only provides +//! handles kernel selection automatically; this module only provides //! the ability to force specific backends for testing/debugging. use crate::platform::Caps; diff --git a/src/checksum/crc24/portable.rs b/src/checksum/crc24/portable.rs index bd2bd86a..0afe0baf 100644 --- a/src/checksum/crc24/portable.rs +++ b/src/checksum/crc24/portable.rs @@ -13,8 +13,7 @@ pub fn crc24_openpgp_slice8(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OpenPGP byte-at-a-time lookup computation (MSB-first). /// -/// This is typically faster than slice-by-8 for tiny buffers because it uses a -/// single 256-entry table. +/// Uses one 256-entry table rather than the slice-by-8 table set. #[inline(always)] #[allow(clippy::indexing_slicing)] // index is 0..=255 by byte cast, table is [u32; 256] pub fn crc24_openpgp_bytewise(crc: u32, data: &[u8]) -> u32 { diff --git a/src/checksum/crc24/power.rs b/src/checksum/crc24/power.rs index 02cb3d56..0e00b74f 100644 --- a/src/checksum/crc24/power.rs +++ b/src/checksum/crc24/power.rs @@ -764,3 +764,46 @@ pub fn crc24_openpgp_vpmsum_8way_safe(crc: u32, data: &[u8]) -> u32 { state = unsafe { crc24_width32_vpmsum_bitrev_8way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; from_reflected_state(state) } + +#[cfg(test)] +mod tests { + extern crate std; + + use alloc::vec::Vec; + + use super::*; + + const LENS: &[usize] = &[0, 1, 7, 15, 16, 31, 63, 64, 127, 128, 255, 256, 1023, 1024, 4096]; + const OFFSETS: &[usize] = &[0, 1, 7, 15]; + const STATES: &[u32] = &[0, 0x00b7_04ce, 0x005a_a5a5, 0x00ff_ffff]; + + #[test] + fn vpmsum_kernels_match_portable() { + if !crate::platform::caps().has(crate::platform::caps::power::VPMSUM_READY) { + return; + } + + let input: Vec = (0..4111) + .map(|i| (i as u8).wrapping_mul(19).wrapping_add((i >> 8) as u8)) + .collect(); + for (name, kernel) in [ + ("openpgp/vpmsum", crc24_openpgp_vpmsum_safe as fn(u32, &[u8]) -> u32), + ("openpgp/vpmsum-2way", crc24_openpgp_vpmsum_2way_safe), + ("openpgp/vpmsum-4way", crc24_openpgp_vpmsum_4way_safe), + ("openpgp/vpmsum-8way", crc24_openpgp_vpmsum_8way_safe), + ] { + for &state in STATES { + for &offset in OFFSETS { + for &len in LENS { + let slice = &input[offset..offset + len]; + assert_eq!( + kernel(state, slice), + super::super::portable::crc24_openpgp_slice8(state, slice), + "{name} state={state:#010x} offset={offset} len={len}" + ); + } + } + } + } + } +} diff --git a/src/checksum/crc24/riscv64.rs b/src/checksum/crc24/riscv64.rs index f1f2c370..5e1bd1a5 100644 --- a/src/checksum/crc24/riscv64.rs +++ b/src/checksum/crc24/riscv64.rs @@ -962,3 +962,66 @@ pub fn crc24_openpgp_zvbc_4way_safe(crc: u32, data: &[u8]) -> u32 { state = unsafe { crc24_width32_zvbc_4way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; from_reflected_state(state) } + +#[cfg(test)] +mod tests { + extern crate std; + + use alloc::vec::Vec; + + use super::*; + + const LENS: &[usize] = &[0, 1, 7, 15, 16, 31, 63, 64, 127, 128, 255, 256, 1023, 1024, 4096]; + const OFFSETS: &[usize] = &[0, 1, 7, 15]; + const STATES: &[u32] = &[0, 0x00b7_04ce, 0x005a_a5a5, 0x00ff_ffff]; + + fn assert_kernel(name: &str, kernel: fn(u32, &[u8]) -> u32) { + let input: Vec = (0..4111) + .map(|i| (i as u8).wrapping_mul(37).wrapping_add((i >> 8) as u8)) + .collect(); + for &state in STATES { + for &offset in OFFSETS { + for &len in LENS { + let slice = &input[offset..offset + len]; + assert_eq!( + kernel(state, slice), + super::super::portable::crc24_openpgp_slice8(state, slice), + "{name} state={state:#010x} offset={offset} len={len}" + ); + } + } + } + } + + #[test] + fn zbc_kernels_match_portable() { + if !crate::platform::caps().has(crate::platform::caps::riscv::ZBC) { + return; + } + + for (name, kernel) in [ + ("openpgp/zbc", crc24_openpgp_zbc_safe as fn(u32, &[u8]) -> u32), + ("openpgp/zbc-2way", crc24_openpgp_zbc_2way_safe), + ("openpgp/zbc-4way", crc24_openpgp_zbc_4way_safe), + ] { + assert_kernel(name, kernel); + } + } + + #[test] + fn zvbc_kernels_match_portable() { + use crate::platform::caps::riscv; + + if !crate::platform::caps().has(riscv::V.union(riscv::ZVBC)) { + return; + } + + for (name, kernel) in [ + ("openpgp/zvbc", crc24_openpgp_zvbc_safe as fn(u32, &[u8]) -> u32), + ("openpgp/zvbc-2way", crc24_openpgp_zvbc_2way_safe), + ("openpgp/zvbc-4way", crc24_openpgp_zvbc_4way_safe), + ] { + assert_kernel(name, kernel); + } + } +} diff --git a/src/checksum/crc24/s390x.rs b/src/checksum/crc24/s390x.rs index ef3bf617..f8b35ca8 100644 --- a/src/checksum/crc24/s390x.rs +++ b/src/checksum/crc24/s390x.rs @@ -494,3 +494,45 @@ pub fn crc24_openpgp_vgfm_4way_safe(crc: u32, data: &[u8]) -> u32 { state = unsafe { crc24_width32_vgfm_bitrev_4way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; from_reflected_state(state) } + +#[cfg(test)] +mod tests { + extern crate std; + + use alloc::vec::Vec; + + use super::*; + + const LENS: &[usize] = &[0, 1, 7, 15, 16, 31, 63, 64, 127, 128, 255, 256, 1023, 1024, 4096]; + const OFFSETS: &[usize] = &[0, 1, 7, 15]; + const STATES: &[u32] = &[0, 0x00b7_04ce, 0x005a_a5a5, 0x00ff_ffff]; + + #[test] + fn vgfm_kernels_match_portable() { + if !crate::platform::caps().has(crate::platform::caps::s390x::VECTOR) { + return; + } + + let input: Vec = (0..4111) + .map(|i| (i as u8).wrapping_mul(47).wrapping_add((i >> 8) as u8)) + .collect(); + for (name, kernel) in [ + ("openpgp/vgfm", crc24_openpgp_vgfm_safe as fn(u32, &[u8]) -> u32), + ("openpgp/vgfm-2way", crc24_openpgp_vgfm_2way_safe), + ("openpgp/vgfm-4way", crc24_openpgp_vgfm_4way_safe), + ] { + for &state in STATES { + for &offset in OFFSETS { + for &len in LENS { + let slice = &input[offset..offset + len]; + assert_eq!( + kernel(state, slice), + super::super::portable::crc24_openpgp_slice8(state, slice), + "{name} state={state:#010x} offset={offset} len={len}" + ); + } + } + } + } + } +} diff --git a/src/checksum/crc24/x86_64.rs b/src/checksum/crc24/x86_64.rs index b1315fb5..934e4929 100644 --- a/src/checksum/crc24/x86_64.rs +++ b/src/checksum/crc24/x86_64.rs @@ -459,7 +459,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_4way( acc[6] = s0[6].fold_16_reflected(c384, acc[6]); acc[7] = s0[7].fold_16_reflected(c384, acc[7]); - for block in &blocks[aligned..] { + for block in &blocks[quad_aligned..] { fold_block_128_reflected_bitrev(&mut acc, block, coeff_128); } @@ -1689,3 +1689,69 @@ pub fn crc24_openpgp_vpclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { }; from_reflected_state(state) } + +#[cfg(test)] +mod tests { + extern crate std; + + use alloc::vec::Vec; + + use super::*; + + const LENS: &[usize] = &[0, 1, 7, 15, 16, 31, 63, 64, 127, 128, 255, 256, 1023, 1024, 4096]; + const OFFSETS: &[usize] = &[0, 1, 7, 15]; + const STATES: &[u32] = &[0, 0x00b7_04ce, 0x005a_a5a5, 0x00ff_ffff]; + + fn assert_kernel(name: &str, kernel: fn(u32, &[u8]) -> u32) { + let input: Vec = (0..4111) + .map(|i| (i as u8).wrapping_mul(61).wrapping_add((i >> 8) as u8)) + .collect(); + for &state in STATES { + for &offset in OFFSETS { + for &len in LENS { + let slice = &input[offset..offset + len]; + assert_eq!( + kernel(state, slice), + super::super::portable::crc24_openpgp_slice8(state, slice), + "{name} state={state:#010x} offset={offset} len={len}" + ); + } + } + } + } + + #[test] + fn pclmul_kernels_match_portable() { + if !crate::platform::caps().has(crate::platform::caps::x86::PCLMUL_READY) { + return; + } + + for (name, kernel) in [ + ("openpgp/pclmul", crc24_openpgp_pclmul_safe as fn(u32, &[u8]) -> u32), + ("openpgp/pclmul-small", crc24_openpgp_pclmul_small_safe), + ("openpgp/pclmul-2way", crc24_openpgp_pclmul_2way_safe), + ("openpgp/pclmul-4way", crc24_openpgp_pclmul_4way_safe), + ("openpgp/pclmul-7way", crc24_openpgp_pclmul_7way_safe), + ("openpgp/pclmul-8way", crc24_openpgp_pclmul_8way_safe), + ] { + assert_kernel(name, kernel); + } + } + + #[test] + fn vpclmul_kernels_match_portable() { + if !crate::platform::caps().has(crate::platform::caps::x86::VPCLMUL_READY) { + return; + } + + for (name, kernel) in [ + ("openpgp/vpclmul", crc24_openpgp_vpclmul_safe as fn(u32, &[u8]) -> u32), + ("openpgp/vpclmul-2way", crc24_openpgp_vpclmul_2way_safe), + ("openpgp/vpclmul-4way", crc24_openpgp_vpclmul_4way_safe), + ("openpgp/vpclmul-7way", crc24_openpgp_vpclmul_7way_safe), + ("openpgp/vpclmul-8way", crc24_openpgp_vpclmul_8way_safe), + ] { + assert_kernel(name, kernel); + } + } +} diff --git a/src/checksum/crc32/config.rs b/src/checksum/crc32/config.rs index 5b397acf..dd6e986f 100644 --- a/src/checksum/crc32/config.rs +++ b/src/checksum/crc32/config.rs @@ -1,7 +1,7 @@ //! CRC-32 runtime configuration. //! //! This module handles force mode selection for CRC-32. The dispatch module -//! handles optimal kernel selection automatically; this module only provides +//! handles kernel selection automatically; this module only provides //! the ability to force specific backends for testing/debugging. use crate::platform::Caps; diff --git a/src/checksum/crc32/portable.rs b/src/checksum/crc32/portable.rs index 5047829e..20cada34 100644 --- a/src/checksum/crc32/portable.rs +++ b/src/checksum/crc32/portable.rs @@ -14,8 +14,7 @@ pub(crate) const BYTEWISE_KERNEL_NAME: &str = "portable/bytewise"; /// CRC-32 (IEEE) byte-at-a-time lookup computation. /// -/// This is typically faster than slice-by-16 for tiny buffers because it uses a -/// single 256-entry table. +/// Uses one 256-entry table rather than the slice-by-16 table set. #[inline(always)] pub fn crc32_bytewise_ieee(crc: u32, data: &[u8]) -> u32 { crc32_bytewise(crc, data, &kernel_tables::IEEE_TABLES_16[0]) @@ -23,8 +22,7 @@ pub fn crc32_bytewise_ieee(crc: u32, data: &[u8]) -> u32 { /// CRC-32C (Castagnoli) byte-at-a-time lookup computation. /// -/// This is typically faster than slice-by-16 for tiny buffers because it uses a -/// single 256-entry table. +/// Uses one 256-entry table rather than the slice-by-16 table set. #[inline(always)] pub fn crc32c_bytewise(crc: u32, data: &[u8]) -> u32 { crc32_bytewise(crc, data, &kernel_tables::CRC32C_TABLES_16[0]) diff --git a/src/checksum/crc64/aarch64.rs b/src/checksum/crc64/aarch64.rs index 2bd81f89..9e9a0c25 100644 --- a/src/checksum/crc64/aarch64.rs +++ b/src/checksum/crc64/aarch64.rs @@ -457,9 +457,8 @@ unsafe fn update_simd_3way( /// 2-way striping with EOR3 folding. /// -/// Combines the ILP benefit of 2-way striping (major) with EOR3's reduced -/// XOR dependency chain (minor). This is the optimal path on Apple M1+ and -/// AWS Graviton3+ for large buffers. +/// Maintains two independent fold streams and uses EOR3 when merging each +/// polynomial product with input data. #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn update_simd_eor3_2way( state: u64, @@ -545,9 +544,8 @@ unsafe fn update_simd_eor3_2way( /// 3-way striping with EOR3 folding. /// -/// Combines the ILP benefit of 3-way striping (major) with EOR3's reduced -/// XOR dependency chain (minor). This is the optimal path on Apple M1+ and -/// AWS Graviton3+ for very large buffers (32KB+). +/// Maintains three independent fold streams and uses EOR3 when merging each +/// polynomial product with input data. #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn update_simd_eor3_3way( state: u64, @@ -753,7 +751,7 @@ unsafe fn crc64_pmull(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants } } -/// PMULL+EOR3 path: uses 3-way XOR for faster folding. +/// PMULL+EOR3 path: uses EOR3 to combine three XOR operands. /// /// Available on ARMv8.2+ with SHA3 extension (Apple M1+, AWS Graviton3+). #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] @@ -778,8 +776,8 @@ unsafe fn crc64_pmull_eor3( /// Small-buffer PMULL path: fold one 16-byte lane at a time. /// -/// This targets the regime where full 128-byte folding has too much setup cost, -/// but PMULL still outperforms table CRC (typically ~16..127 bytes depending on CPU). +/// This avoids constructing the full 128-byte fold state when only complete +/// 16-byte lanes are available. #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc64_pmull_small( mut state: u64, @@ -859,8 +857,8 @@ pub(crate) unsafe fn crc64_xz_pmull_small(crc: u64, data: &[u8]) -> u64 { /// CRC-64-XZ using a tuned "SVE2 PMULL" tier (2-way striping). /// -/// This is still implemented with NEON+PMULL intrinsics, but is intended for -/// high-throughput Armv9/SVE2-class CPUs where additional ILP helps. +/// This is implemented with NEON+PMULL intrinsics and exposes two independent +/// fold streams for the SVE2-class dispatch tier. /// /// # Safety /// @@ -874,8 +872,8 @@ pub(crate) unsafe fn crc64_xz_sve2_pmull_2way(crc: u64, data: &[u8]) -> u64 { /// CRC-64-XZ using a tuned "SVE2 PMULL" tier (3-way striping). /// -/// This is still implemented with NEON+PMULL intrinsics, but is intended for -/// high-throughput Armv9/SVE2-class CPUs where additional ILP helps. +/// This is implemented with NEON+PMULL intrinsics and exposes three independent +/// fold streams for the SVE2-class dispatch tier. /// /// # Safety /// @@ -929,8 +927,8 @@ pub(crate) unsafe fn crc64_nvme_pmull_small(crc: u64, data: &[u8]) -> u64 { /// CRC-64-NVME using a tuned "SVE2 PMULL" tier (2-way striping). /// -/// This is still implemented with NEON+PMULL intrinsics, but is intended for -/// high-throughput Armv9/SVE2-class CPUs where additional ILP helps. +/// This is implemented with NEON+PMULL intrinsics and exposes two independent +/// fold streams for the SVE2-class dispatch tier. /// /// # Safety /// @@ -944,8 +942,8 @@ pub(crate) unsafe fn crc64_nvme_sve2_pmull_2way(crc: u64, data: &[u8]) -> u64 { /// CRC-64-NVME using a tuned "SVE2 PMULL" tier (3-way striping). /// -/// This is still implemented with NEON+PMULL intrinsics, but is intended for -/// high-throughput Armv9/SVE2-class CPUs where additional ILP helps. +/// This is implemented with NEON+PMULL intrinsics and exposes three independent +/// fold streams for the SVE2-class dispatch tier. /// /// # Safety /// @@ -1009,9 +1007,7 @@ pub(crate) unsafe fn crc64_nvme_pmull_eor3(crc: u64, data: &[u8]) -> u64 { /// CRC-64-XZ using PMULL+EOR3 folding with 2-way striping. /// -/// Combines the ILP benefit of 2-way striping with EOR3's reduced -/// XOR dependency chain. This is optimal for Apple M1+ and AWS Graviton3+ -/// for large buffers. +/// Maintains two independent fold streams and uses EOR3 for three-operand XORs. /// /// # Safety /// @@ -1034,9 +1030,7 @@ pub(crate) unsafe fn crc64_xz_pmull_eor3_2way(crc: u64, data: &[u8]) -> u64 { /// CRC-64-XZ using PMULL+EOR3 folding with 3-way striping. /// -/// Combines the ILP benefit of 3-way striping with EOR3's reduced -/// XOR dependency chain. This is optimal for Apple M1+ and AWS Graviton3+ -/// for very large buffers (32KB+). +/// Maintains three independent fold streams and uses EOR3 for three-operand XORs. /// /// # Safety /// @@ -1060,9 +1054,7 @@ pub(crate) unsafe fn crc64_xz_pmull_eor3_3way(crc: u64, data: &[u8]) -> u64 { /// CRC-64-NVME using PMULL+EOR3 folding with 2-way striping. /// -/// Combines the ILP benefit of 2-way striping with EOR3's reduced -/// XOR dependency chain. This is optimal for Apple M1+ and AWS Graviton3+ -/// for large buffers. +/// Maintains two independent fold streams and uses EOR3 for three-operand XORs. /// /// # Safety /// @@ -1085,9 +1077,7 @@ pub(crate) unsafe fn crc64_nvme_pmull_eor3_2way(crc: u64, data: &[u8]) -> u64 { /// CRC-64-NVME using PMULL+EOR3 folding with 3-way striping. /// -/// Combines the ILP benefit of 3-way striping with EOR3's reduced -/// XOR dependency chain. This is optimal for Apple M1+ and AWS Graviton3+ -/// for very large buffers (32KB+). +/// Maintains three independent fold streams and uses EOR3 for three-operand XORs. /// /// # Safety /// @@ -1658,7 +1648,7 @@ mod tests { return; } - // EOR3 should produce identical results to standard PMULL - just faster + // EOR3 changes only the three-operand XOR lowering. for len in [128, 256, 512, 1024, 4096] { let data = make_data(len); let pmull = crc64_xz_pmull_safe(!0, &data); @@ -1673,7 +1663,7 @@ mod tests { return; } - // EOR3 should produce identical results to standard PMULL - just faster + // EOR3 changes only the three-operand XOR lowering. for len in [128, 256, 512, 1024, 4096] { let data = make_data(len); let pmull = crc64_nvme_pmull_safe(!0, &data); diff --git a/src/checksum/crc64/config.rs b/src/checksum/crc64/config.rs index e6865b2f..3fa7e4a3 100644 --- a/src/checksum/crc64/config.rs +++ b/src/checksum/crc64/config.rs @@ -1,7 +1,7 @@ //! CRC-64 runtime configuration. //! //! This module handles force mode selection for CRC-64. The dispatch module -//! handles optimal kernel selection automatically; this module only provides +//! handles kernel selection automatically; this module only provides //! the ability to force specific backends for testing/debugging. use crate::platform::Caps; diff --git a/src/checksum/crc64/mod.rs b/src/checksum/crc64/mod.rs index 92d5e4f5..4954d8cc 100644 --- a/src/checksum/crc64/mod.rs +++ b/src/checksum/crc64/mod.rs @@ -154,7 +154,7 @@ pub(crate) fn diag_crc64_xz(len: usize) -> Crc64SelectionDiag { effective_force: cfg.effective_force, policy_family: "dispatch", selected_kernel, - selected_streams: 1, // Dispatch uses pre-computed optimal kernel + selected_streams: 1, portable_to_clmul: table.boundaries[0], // xs_max boundary pclmul_to_vpclmul: table.boundaries[2], // m_max boundary small_kernel_max_bytes: table.boundaries[1], // s_max boundary @@ -251,8 +251,7 @@ fn crc64_nvme_reference(crc: u64, data: &[u8]) -> u64 { /// Default SIMD threshold for buffered CRC. /// /// Buffered CRC uses this to decide when to flush accumulated small updates. -/// The dispatch system handles optimal kernel selection; this is a conservative -/// threshold for buffer flush decisions. +/// Kernel selection remains the dispatch table's responsibility. #[cfg(feature = "alloc")] const CRC64_BUFFERED_THRESHOLD: usize = 64; @@ -557,22 +556,9 @@ define_crc_dispatch! { /// - **Final XOR**: 0xFFFFFFFFFFFFFFFF /// - **Reflect input/output**: Yes /// -/// # Performance Notes -/// -/// For optimal throughput, prefer larger updates when possible: -/// -/// | Update Size | Path | Notes | -/// |-------------|------|-------| -/// | < 32-128 bytes | Portable slice-by-8 | Threshold varies by CPU | -/// | ≥ 32-128 bytes | SIMD (PCLMULQDQ/PMULL) | Hardware accelerated | -/// -/// The exact threshold is microarchitecture-specific: -/// - AMD Zen 4/5: 32 bytes (fast SIMD setup) -/// - Intel SPR: 128 bytes (ZMM warmup overhead) -/// - Apple M1-M5: 48 bytes (efficient PMULL) -/// -/// For streaming many small chunks, consider using [`Crc64::buffered`] which -/// accumulates data internally until reaching the SIMD threshold. +/// Runtime dispatch selects a kernel from the current platform table. For +/// streaming many short fragments, [`Crc64::buffered`] coalesces updates before +/// dispatch; measure the actual workload before assuming that buffering helps. /// /// # Examples /// @@ -734,22 +720,10 @@ impl Crc64 { /// - **Final XOR**: 0xFFFFFFFFFFFFFFFF /// - **Reflect input/output**: Yes /// -/// # Performance Notes -/// -/// For optimal throughput, prefer larger updates when possible: -/// -/// | Update Size | Path | Notes | -/// |-------------|------|-------| -/// | < 32-128 bytes | Portable slice-by-8 | Threshold varies by CPU | -/// | ≥ 32-128 bytes | SIMD (PCLMULQDQ/PMULL) | Hardware accelerated | -/// -/// The exact threshold is microarchitecture-specific: -/// - AMD Zen 4/5: 32 bytes (fast SIMD setup) -/// - Intel SPR: 128 bytes (ZMM warmup overhead) -/// - Apple M1-M5: 48 bytes (efficient PMULL) -/// -/// For streaming many small chunks, consider using [`Crc64Nvme::buffered`] which -/// accumulates data internally until reaching the SIMD threshold. +/// Runtime dispatch selects a kernel from the current platform table. For +/// streaming many short fragments, [`Crc64Nvme::buffered`] coalesces updates +/// before dispatch; measure the actual workload before assuming that buffering +/// helps. /// /// # Examples /// @@ -896,7 +870,7 @@ define_buffered_crc! { /// /// Use when you expect many small updates (< 64 bytes). This wrapper /// accumulates data internally until reaching the SIMD threshold, then - /// flushes in batches for optimal throughput. + /// flushes in batches before calling the active kernel. /// /// # When to Use /// @@ -934,7 +908,7 @@ define_buffered_crc! { /// /// Use when you expect many small updates (< 64 bytes). This wrapper /// accumulates data internally until reaching the SIMD threshold, then - /// flushes in batches for optimal throughput. + /// flushes in batches before calling the active kernel. /// /// # When to Use /// diff --git a/src/checksum/crc64/portable.rs b/src/checksum/crc64/portable.rs index f1e0c22e..1d9bb895 100644 --- a/src/checksum/crc64/portable.rs +++ b/src/checksum/crc64/portable.rs @@ -40,8 +40,7 @@ pub fn crc64_slice16_nvme(crc: u64, data: &[u8]) -> u64 { /// CRC-64-XZ byte-at-a-time lookup computation. /// -/// This is typically faster than slice-by-16 for tiny buffers because it uses a -/// single 256-entry table. +/// Uses one 256-entry table rather than the slice-by-16 table set. #[inline(always)] pub fn crc64_xz_bytewise(crc: u64, data: &[u8]) -> u64 { crc64_bytewise(crc, data, &kernel_tables::XZ_TABLES_16[0]) @@ -49,8 +48,7 @@ pub fn crc64_xz_bytewise(crc: u64, data: &[u8]) -> u64 { /// CRC-64-NVME byte-at-a-time lookup computation. /// -/// This is typically faster than slice-by-16 for tiny buffers because it uses a -/// single 256-entry table. +/// Uses one 256-entry table rather than the slice-by-16 table set. #[inline(always)] pub fn crc64_nvme_bytewise(crc: u64, data: &[u8]) -> u64 { crc64_bytewise(crc, data, &kernel_tables::NVME_TABLES_16[0]) diff --git a/src/checksum/crc64/power.rs b/src/checksum/crc64/power.rs index 4187bc3a..7451295b 100644 --- a/src/checksum/crc64/power.rs +++ b/src/checksum/crc64/power.rs @@ -1087,3 +1087,59 @@ pub fn crc64_nvme_vpmsum_8way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_nvme_vpmsum_8way(crc, data) } } + +#[cfg(test)] +mod tests { + extern crate std; + + use alloc::vec::Vec; + + use super::*; + + const LENS: &[usize] = &[0, 1, 7, 15, 16, 31, 63, 64, 127, 128, 255, 256, 1023, 1024, 4096]; + const OFFSETS: &[usize] = &[0, 1, 7, 15]; + const STATES: &[u64] = &[0, 0x0123_4567_89ab_cdef, 0xa5a5_5a5a_dead_beef, u64::MAX]; + + fn assert_kernel(name: &str, kernel: fn(u64, &[u8]) -> u64, portable: fn(u64, &[u8]) -> u64) { + let input: Vec = (0..4111) + .map(|i| (i as u8).wrapping_mul(23).wrapping_add((i >> 8) as u8)) + .collect(); + for &state in STATES { + for &offset in OFFSETS { + for &len in LENS { + let slice = &input[offset..offset + len]; + assert_eq!( + kernel(state, slice), + portable(state, slice), + "{name} state={state:#018x} offset={offset} len={len}" + ); + } + } + } + } + + #[test] + fn vpmsum_kernels_match_portable() { + if !crate::platform::caps().has(crate::platform::caps::power::VPMSUM_READY) { + return; + } + + for (name, kernel) in [ + ("xz/vpmsum", crc64_xz_vpmsum_safe as fn(u64, &[u8]) -> u64), + ("xz/vpmsum-2way", crc64_xz_vpmsum_2way_safe), + ("xz/vpmsum-4way", crc64_xz_vpmsum_4way_safe), + ("xz/vpmsum-8way", crc64_xz_vpmsum_8way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc64_slice16_xz); + } + + for (name, kernel) in [ + ("nvme/vpmsum", crc64_nvme_vpmsum_safe as fn(u64, &[u8]) -> u64), + ("nvme/vpmsum-2way", crc64_nvme_vpmsum_2way_safe), + ("nvme/vpmsum-4way", crc64_nvme_vpmsum_4way_safe), + ("nvme/vpmsum-8way", crc64_nvme_vpmsum_8way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc64_slice16_nvme); + } + } +} diff --git a/src/checksum/crc64/riscv64.rs b/src/checksum/crc64/riscv64.rs index 41d474c3..011ad1d5 100644 --- a/src/checksum/crc64/riscv64.rs +++ b/src/checksum/crc64/riscv64.rs @@ -1464,3 +1464,84 @@ pub fn crc64_nvme_zvbc_4way_safe(crc: u64, data: &[u8]) -> u64 { // buffers and register-only operands are established before this block. unsafe { crc64_nvme_zvbc_4way(crc, data) } } + +#[cfg(test)] +mod tests { + extern crate std; + + use alloc::vec::Vec; + + use super::*; + + const LENS: &[usize] = &[0, 1, 7, 15, 16, 31, 63, 64, 127, 128, 255, 256, 1023, 1024, 4096]; + const OFFSETS: &[usize] = &[0, 1, 7, 15]; + const STATES: &[u64] = &[0, 0x0123_4567_89ab_cdef, 0xa5a5_5a5a_dead_beef, u64::MAX]; + + fn assert_kernel(name: &str, kernel: fn(u64, &[u8]) -> u64, portable: fn(u64, &[u8]) -> u64) { + let input: Vec = (0..4111) + .map(|i| (i as u8).wrapping_mul(41).wrapping_add((i >> 8) as u8)) + .collect(); + for &state in STATES { + for &offset in OFFSETS { + for &len in LENS { + let slice = &input[offset..offset + len]; + assert_eq!( + kernel(state, slice), + portable(state, slice), + "{name} state={state:#018x} offset={offset} len={len}" + ); + } + } + } + } + + #[test] + fn zbc_kernels_match_portable() { + if !crate::platform::caps().has(crate::platform::caps::riscv::ZBC) { + return; + } + + for (name, kernel) in [ + ("xz/zbc", crc64_xz_zbc_safe as fn(u64, &[u8]) -> u64), + ("xz/zbc-2way", crc64_xz_zbc_2way_safe), + ("xz/zbc-4way", crc64_xz_zbc_4way_safe), + ("xz/zbc-8way", crc64_xz_zbc_8way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc64_slice16_xz); + } + + for (name, kernel) in [ + ("nvme/zbc", crc64_nvme_zbc_safe as fn(u64, &[u8]) -> u64), + ("nvme/zbc-2way", crc64_nvme_zbc_2way_safe), + ("nvme/zbc-4way", crc64_nvme_zbc_4way_safe), + ("nvme/zbc-8way", crc64_nvme_zbc_8way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc64_slice16_nvme); + } + } + + #[test] + fn zvbc_kernels_match_portable() { + use crate::platform::caps::riscv; + + if !crate::platform::caps().has(riscv::V.union(riscv::ZVBC)) { + return; + } + + for (name, kernel) in [ + ("xz/zvbc", crc64_xz_zvbc_safe as fn(u64, &[u8]) -> u64), + ("xz/zvbc-2way", crc64_xz_zvbc_2way_safe), + ("xz/zvbc-4way", crc64_xz_zvbc_4way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc64_slice16_xz); + } + + for (name, kernel) in [ + ("nvme/zvbc", crc64_nvme_zvbc_safe as fn(u64, &[u8]) -> u64), + ("nvme/zvbc-2way", crc64_nvme_zvbc_2way_safe), + ("nvme/zvbc-4way", crc64_nvme_zvbc_4way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc64_slice16_nvme); + } + } +} diff --git a/src/checksum/crc64/s390x.rs b/src/checksum/crc64/s390x.rs index b6087da2..c806e28d 100644 --- a/src/checksum/crc64/s390x.rs +++ b/src/checksum/crc64/s390x.rs @@ -648,3 +648,57 @@ pub fn crc64_nvme_vgfm_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc64_nvme_vgfm_4way(crc, data) } } + +#[cfg(test)] +mod tests { + extern crate std; + + use alloc::vec::Vec; + + use super::*; + + const LENS: &[usize] = &[0, 1, 7, 15, 16, 31, 63, 64, 127, 128, 255, 256, 1023, 1024, 4096]; + const OFFSETS: &[usize] = &[0, 1, 7, 15]; + const STATES: &[u64] = &[0, 0x0123_4567_89ab_cdef, 0xa5a5_5a5a_dead_beef, u64::MAX]; + + fn assert_kernel(name: &str, kernel: fn(u64, &[u8]) -> u64, portable: fn(u64, &[u8]) -> u64) { + let input: Vec = (0..4111) + .map(|i| (i as u8).wrapping_mul(53).wrapping_add((i >> 8) as u8)) + .collect(); + for &state in STATES { + for &offset in OFFSETS { + for &len in LENS { + let slice = &input[offset..offset + len]; + assert_eq!( + kernel(state, slice), + portable(state, slice), + "{name} state={state:#018x} offset={offset} len={len}" + ); + } + } + } + } + + #[test] + fn vgfm_kernels_match_portable() { + if !crate::platform::caps().has(crate::platform::caps::s390x::VECTOR) { + return; + } + + for (name, kernel) in [ + ("xz/vgfm", crc64_xz_vgfm_safe as fn(u64, &[u8]) -> u64), + ("xz/vgfm-2way", crc64_xz_vgfm_2way_safe), + ("xz/vgfm-4way", crc64_xz_vgfm_4way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc64_slice16_xz); + } + + for (name, kernel) in [ + ("nvme/vgfm", crc64_nvme_vgfm_safe as fn(u64, &[u8]) -> u64), + ("nvme/vgfm-2way", crc64_nvme_vgfm_2way_safe), + ("nvme/vgfm-4way", crc64_nvme_vgfm_4way_safe), + ] { + assert_kernel(name, kernel, super::super::portable::crc64_slice16_nvme); + } + } +} diff --git a/src/checksum/crc64/x86_64.rs b/src/checksum/crc64/x86_64.rs index 25c03670..68d625f9 100644 --- a/src/checksum/crc64/x86_64.rs +++ b/src/checksum/crc64/x86_64.rs @@ -646,8 +646,8 @@ unsafe fn crc64_pclmul_8way( /// Small-buffer CLMUL path: fold one 16-byte lane at a time. /// -/// This targets the regime where full 128-byte folding has too much setup cost, -/// but CLMUL still outperforms table CRC (typically ~16..127 bytes depending on CPU). +/// This avoids constructing the full 128-byte fold state when only complete +/// 16-byte lanes are available. #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn crc64_pclmul_small( mut state: u64, @@ -1443,12 +1443,11 @@ unsafe fn finalize_4x512_state( /// 4×512-bit VPCLMUL kernel with double-unrolling and software prefetch. /// -/// Processes 512 bytes per iteration (2 × 4 × 64-byte __m512i registers), -/// folding at 2048-bit distance for maximum throughput on Ice Lake+/Zen4+. +/// Processes 512 bytes per iteration (2 × 4 × 64-byte `__m512i` registers) +/// and folds each stream at 2,048-bit distance. /// -/// The double-unroll reduces loop overhead by 50% and improves instruction-level -/// parallelism. Software prefetch hints ensure data arrives in L1 cache before -/// the CPU needs it, hiding memory latency. +/// The double-unroll halves the number of loop-control steps. Software +/// prefetch remains a target-dependent hint. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn crc64_vpclmul_4x512( mut state: u64, @@ -1883,8 +1882,7 @@ pub(crate) unsafe fn crc64_xz_vpclmul_8way(crc: u64, data: &[u8]) -> u64 { /// CRC-64-XZ using VPCLMULQDQ (4×512-bit variant). /// -/// High-throughput path processing 256 bytes per iteration. -/// Optimal for large buffers (≥256 bytes) on Ice Lake+/Zen4+ CPUs. +/// Processes 512 bytes per main-loop iteration across four fold streams. /// /// # Safety /// @@ -2135,8 +2133,7 @@ pub(crate) unsafe fn crc64_nvme_vpclmul_8way(crc: u64, data: &[u8]) -> u64 { /// CRC-64-NVME using VPCLMULQDQ (4×512-bit variant). /// -/// High-throughput path processing 256 bytes per iteration. -/// Optimal for large buffers (≥256 bytes) on Ice Lake+/Zen4+ CPUs. +/// Processes 512 bytes per main-loop iteration across four fold streams. /// /// # Safety /// diff --git a/src/checksum/io.rs b/src/checksum/io.rs index 080e3060..06ab302d 100644 --- a/src/checksum/io.rs +++ b/src/checksum/io.rs @@ -6,7 +6,7 @@ //! //! # Performance //! -//! - Zero-cost abstraction: All methods are `#[inline]` +//! - Inline-friendly methods over the checksum traits //! - Vectored I/O support: Uses the `update_vectored` method when available //! - Correctness: Only hashes bytes actually transferred (handles short reads/writes) //! diff --git a/src/checksum/kernel_table.rs b/src/checksum/kernel_table.rs index e1c64a52..974679e3 100644 --- a/src/checksum/kernel_table.rs +++ b/src/checksum/kernel_table.rs @@ -1,8 +1,7 @@ //! Internal CRC kernel tables and one-shot helpers. //! //! The public `checksum::dispatch` API was removed. What remains here is the -//! internal table-driven selector that the CRC implementations use after the -//! tuning work was collapsed into static, benchmark-backed kernel tables. +//! internal table-driven selector and its manually maintained kernel choices. #[cfg(feature = "crc16")] use crate::checksum::dispatchers::Crc16Fn; @@ -47,15 +46,11 @@ pub(crate) fn active_crc64_table() -> &'static KernelTable { /// Maximum input size for the inline bytewise fast-path. /// /// Inputs at or below this size bypass all dispatch machinery and use a simple -/// byte-at-a-time table lookup. This eliminates `active_table()` + `select_fns()` -/// + indirect fn ptr overhead (~7-10 ns) that dominates at tiny sizes. -/// -/// Set to 7 (not 64) because at 8+ bytes the dispatch path's slice-by-N or -/// hardware-accelerated kernels outperform the bytewise loop. +/// byte-at-a-time table lookup. Larger inputs use normal table dispatch. const FAST_PATH_MAX: usize = 7; -// `core::hint::cold_path` is 1.95+; a `#[cold]` empty fn produces identical -// LLVM branch-weight metadata and keeps MSRV at 1.93. +// `core::hint::cold_path` is newer than the crate's supported Rust baseline; +// a `#[cold]` empty function produces the same branch-weight metadata. #[cold] #[inline] fn cold_path() {} @@ -111,8 +106,7 @@ pub(crate) fn crc32_ieee(data: &[u8]) -> u32 { // - Medium (128–1024 B): PMULL v12e_v1 fold — 12-lane carryless multiply amortizes setup cost. // Inlined to eliminate indirect-call barrier. // - Large (>1024 B + EOR3): v9s3x2e_s3 EOR3 fusion — 9 PMULL lanes interleaved with 3 scalar CRC - // streams for ILP overlap; `veor3q_u64` reduces XOR chains. Significantly faster than v12e_v1 at - // scale. + // streams for ILP overlap; `veor3q_u64` reduces the XOR chains. #[cfg(target_arch = "aarch64")] { use crate::platform::caps::aarch64; @@ -380,7 +374,7 @@ impl KernelSet { /// Complete kernel table for one platform. /// -/// Contains pre-selected optimal kernels for each (variant, size_class) pair. +/// Contains the selected kernels for each `(variant, size_class)` pair. /// Size class boundaries define when to transition between kernel tiers. /// /// Hot-path function pointers ([`KernelFnSet`]) are stored separately from @@ -780,26 +774,9 @@ mod aarch64_tables { } } - // Apple M1-M3 Table + // Apple M1-M3 table. // - // Benchmark source: macOS local (2026-01-20) // Features: PMULL + SHA3 (EOR3) - // Peak throughputs: CRC-16 ~60 GiB/s, CRC-32 ~75 GiB/s, CRC-64 ~62 GiB/s - // - // Optimal kernels per (variant, peak): - // crc16/ccitt: pmull, streams=3, 60.15 GiB/s - // crc16/ibm: pmull, streams=3, 58.77 GiB/s - // crc24/openpgp: pmull, streams=3, 44.70 GiB/s - // crc32/ieee: pmull-eor3-v9s3x2e-s3, streams=1, 74.32 GiB/s - // crc32c: pmull-eor3-v9s3x2e-s3, streams=1, 75.32 GiB/s - // crc64/xz: pmull, streams=3, 62.58 GiB/s - // crc64/nvme: pmull-eor3, streams=3, 62.57 GiB/s - // AppleM1M3 Table - // - // Generated from benchmark-derived dispatch data. Do not edit manually. - // AppleM1M3 Table - // - // Generated from benchmark-derived dispatch data. Do not edit manually. pub static APPLE_M1M3_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::aarch64::CRC_READY .union(crate::platform::caps::aarch64::PMULL_EOR3_READY) @@ -931,24 +908,9 @@ mod aarch64_tables { }, }; - // Graviton2 Table + // Graviton2 table. // - // Benchmark source: Namespace linux-arm64 runner (2026-01-20) // Features: PMULL (no EOR3/SHA3) - // Peak throughputs: CRC-16 ~33 GiB/s, CRC-32 ~40 GiB/s, CRC-64 ~33 GiB/s - // - // Optimal kernels per (variant, peak): - // crc16/ccitt: pmull, streams=1, 33.42 GiB/s - // crc16/ibm: pmull, streams=1, 33.45 GiB/s - // crc24/openpgp: pmull, streams=1, 24.85 GiB/s - // crc32/ieee: pmull-eor3-v9s3x2e-s3, streams=1, 40.31 GiB/s - // crc32c: pmull-eor3-v9s3x2e-s3, streams=1, 40.11 GiB/s - // crc64/xz: pmull-eor3, streams=1, 33.49 GiB/s - // crc64/nvme: pmull, streams=1, 33.33 GiB/s - // - // Note: Graviton2 benchmark shows pmull-eor3 winning for CRC-64/XZ even - // without SHA3 feature flag - the EOR3 instruction is available through - // a different path on this hardware. #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] pub static GRAVITON2_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::aarch64::CRC_READY.union(crate::platform::caps::aarch64::PMULL_EOR3_READY), @@ -968,11 +930,11 @@ mod aarch64_tables { #[cfg(feature = "crc24")] crc24_openpgp_name: "aarch64/pmull-small", #[cfg(feature = "crc32")] - crc32_ieee: crc32_k::CRC32_PMULL_SMALL_KERNEL, // pmull-small beats hwcrc @ 9.53 GiB/s + crc32_ieee: crc32_k::CRC32_PMULL_SMALL_KERNEL, #[cfg(feature = "crc32")] crc32_ieee_name: "aarch64/pmull-small", #[cfg(feature = "crc32")] - crc32c: crc32_k::CRC32C_PMULL_SMALL_KERNEL, // pmull-small beats hwcrc @ 9.54 GiB/s + crc32c: crc32_k::CRC32C_PMULL_SMALL_KERNEL, #[cfg(feature = "crc32")] crc32c_name: "aarch64/pmull-small", #[cfg(feature = "crc64")] @@ -995,127 +957,121 @@ mod aarch64_tables { #[cfg(feature = "crc16")] crc16_ibm_name: "aarch64/pmull-small", #[cfg(feature = "crc24")] - crc24_openpgp: crc24_k::OPENPGP_PMULL[0], // 1-way @ 11.08 GiB/s + crc24_openpgp: crc24_k::OPENPGP_PMULL[0], #[cfg(feature = "crc24")] crc24_openpgp_name: "aarch64/pmull", #[cfg(feature = "crc32")] - crc32_ieee: crc32_k::CRC32_PMULL_SMALL_KERNEL, // pmull-small @ 13.30 GiB/s + crc32_ieee: crc32_k::CRC32_PMULL_SMALL_KERNEL, #[cfg(feature = "crc32")] crc32_ieee_name: "aarch64/pmull-small", #[cfg(feature = "crc32")] - crc32c: crc32_k::CRC32C_PMULL_SMALL_KERNEL, // pmull-small @ 13.30 GiB/s + crc32c: crc32_k::CRC32C_PMULL_SMALL_KERNEL, #[cfg(feature = "crc32")] crc32c_name: "aarch64/pmull-small", #[cfg(feature = "crc64")] - crc64_xz: crc64_k::XZ_PMULL[0], // pmull @ 13.60 GiB/s + crc64_xz: crc64_k::XZ_PMULL[0], #[cfg(feature = "crc64")] crc64_xz_name: "aarch64/pmull", #[cfg(feature = "crc64")] - crc64_nvme: crc64_k::NVME_PMULL[0], // pmull @ 13.58 GiB/s (bench: pmull beats pmull-eor3) + crc64_nvme: crc64_k::NVME_PMULL[0], #[cfg(feature = "crc64")] crc64_nvme_name: "aarch64/pmull", }, m: KernelSet { #[cfg(feature = "crc16")] - crc16_ccitt: crc16_k::CCITT_PMULL[0], // 1-way @ 29.19 GiB/s + crc16_ccitt: crc16_k::CCITT_PMULL[0], #[cfg(feature = "crc16")] crc16_ccitt_name: "aarch64/pmull", #[cfg(feature = "crc16")] - crc16_ibm: crc16_k::IBM_PMULL[0], // 1-way @ 29.20 GiB/s + crc16_ibm: crc16_k::IBM_PMULL[0], #[cfg(feature = "crc16")] crc16_ibm_name: "aarch64/pmull", #[cfg(feature = "crc24")] - crc24_openpgp: crc24_k::OPENPGP_PMULL[0], // 1-way @ 19.95 GiB/s + crc24_openpgp: crc24_k::OPENPGP_PMULL[0], #[cfg(feature = "crc24")] crc24_openpgp_name: "aarch64/pmull", #[cfg(feature = "crc32")] - crc32_ieee: crc32_k::CRC32_PMULL_SMALL_KERNEL, // pmull-small @ 28.62 GiB/s + crc32_ieee: crc32_k::CRC32_PMULL_SMALL_KERNEL, #[cfg(feature = "crc32")] crc32_ieee_name: "aarch64/pmull-small", #[cfg(feature = "crc32")] - crc32c: crc32_k::CRC32C_PMULL_SMALL_KERNEL, // pmull-small @ 28.68 GiB/s + crc32c: crc32_k::CRC32C_PMULL_SMALL_KERNEL, #[cfg(feature = "crc32")] crc32c_name: "aarch64/pmull-small", #[cfg(feature = "crc64")] - crc64_xz: crc64_k::XZ_PMULL[0], // pmull @ 30.39 GiB/s (pmull-eor3 slower here) + crc64_xz: crc64_k::XZ_PMULL[0], #[cfg(feature = "crc64")] crc64_xz_name: "aarch64/pmull", #[cfg(feature = "crc64")] - crc64_nvme: crc64_k::NVME_PMULL[0], // pmull @ 30.45 GiB/s + crc64_nvme: crc64_k::NVME_PMULL[0], #[cfg(feature = "crc64")] crc64_nvme_name: "aarch64/pmull", }, l: KernelSet { #[cfg(feature = "crc16")] - crc16_ccitt: crc16_k::CCITT_PMULL[0], // 1-way @ 33.01 GiB/s + crc16_ccitt: crc16_k::CCITT_PMULL[0], #[cfg(feature = "crc16")] crc16_ccitt_name: "aarch64/pmull", #[cfg(feature = "crc16")] - crc16_ibm: crc16_k::IBM_PMULL[0], // 1-way @ 32.78 GiB/s + crc16_ibm: crc16_k::IBM_PMULL[0], #[cfg(feature = "crc16")] crc16_ibm_name: "aarch64/pmull", #[cfg(feature = "crc24")] - crc24_openpgp: crc24_k::OPENPGP_PMULL[0], // 1-way @ 24.88 GiB/s + crc24_openpgp: crc24_k::OPENPGP_PMULL[0], #[cfg(feature = "crc24")] crc24_openpgp_name: "aarch64/pmull", #[cfg(feature = "crc32")] - crc32_ieee: crc32_k::CRC32_PMULL_EOR3[0], // pmull-eor3-v9s3x2e-s3 @ 39.82 GiB/s + crc32_ieee: crc32_k::CRC32_PMULL_EOR3[0], #[cfg(feature = "crc32")] crc32_ieee_name: "aarch64/pmull-eor3-v9s3x2e-s3", #[cfg(feature = "crc32")] - crc32c: crc32_k::CRC32C_PMULL_EOR3[0], // pmull-eor3-v9s3x2e-s3 @ 39.93 GiB/s + crc32c: crc32_k::CRC32C_PMULL_EOR3[0], #[cfg(feature = "crc32")] crc32c_name: "aarch64/pmull-eor3-v9s3x2e-s3", #[cfg(feature = "crc64")] - crc64_xz: crc64_k::XZ_PMULL_EOR3[0], // pmull-eor3 @ 33.07 GiB/s + crc64_xz: crc64_k::XZ_PMULL_EOR3[0], #[cfg(feature = "crc64")] crc64_xz_name: "aarch64/pmull-eor3", #[cfg(feature = "crc64")] - crc64_nvme: crc64_k::NVME_PMULL[0], // pmull @ 33.08 GiB/s + crc64_nvme: crc64_k::NVME_PMULL[0], #[cfg(feature = "crc64")] crc64_nvme_name: "aarch64/pmull", }, }; - // Graviton3 Table + // Graviton3 table. // - // Benchmark source: `src/checksum/bench_baseline/linux_arm64_graviton3_kernels.txt` // Features: PMULL + SHA3/EOR3 - // Peak throughputs: CRC-16 ~38 GiB/s, CRC-32 ~46 GiB/s, CRC-64 ~38 GiB/s - // - // Key differences vs Graviton2: - // - Higher throughput (~25% faster across the board) - // - Different optimal kernel choices for CRC16@s and CRC64/NVME #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] const G3_XS: KernelSet = KernelSet { #[cfg(feature = "crc16")] - crc16_ccitt: crc16_k::CCITT_PMULL_SMALL_KERNEL, // pmull-small @ 8.01 GiB/s + crc16_ccitt: crc16_k::CCITT_PMULL_SMALL_KERNEL, #[cfg(feature = "crc16")] crc16_ccitt_name: "aarch64/pmull-small", #[cfg(feature = "crc16")] - crc16_ibm: crc16_k::IBM_PMULL_SMALL_KERNEL, // pmull-small @ 7.85 GiB/s + crc16_ibm: crc16_k::IBM_PMULL_SMALL_KERNEL, #[cfg(feature = "crc16")] crc16_ibm_name: "aarch64/pmull-small", #[cfg(feature = "crc24")] - crc24_openpgp: crc24_k::OPENPGP_PMULL_SMALL_KERNEL, // pmull-small @ 6.81 GiB/s + crc24_openpgp: crc24_k::OPENPGP_PMULL_SMALL_KERNEL, #[cfg(feature = "crc24")] crc24_openpgp_name: "aarch64/pmull-small", #[cfg(feature = "crc32")] - crc32_ieee: crc32_k::CRC32_PMULL_SMALL_KERNEL, // pmull-small @ 10.30 GiB/s + crc32_ieee: crc32_k::CRC32_PMULL_SMALL_KERNEL, #[cfg(feature = "crc32")] crc32_ieee_name: "aarch64/pmull-small", #[cfg(feature = "crc32")] - crc32c: crc32_k::CRC32C_PMULL_SMALL_KERNEL, // pmull-small @ 12.49 GiB/s + crc32c: crc32_k::CRC32C_PMULL_SMALL_KERNEL, #[cfg(feature = "crc32")] crc32c_name: "aarch64/pmull-small", #[cfg(feature = "crc64")] - crc64_xz: crc64_k::XZ_PMULL_SMALL, // pmull-small @ 7.06 GiB/s + crc64_xz: crc64_k::XZ_PMULL_SMALL, #[cfg(feature = "crc64")] crc64_xz_name: "aarch64/pmull-small", #[cfg(feature = "crc64")] - crc64_nvme: crc64_k::NVME_PMULL_SMALL, // pmull-small @ 7.06 GiB/s + crc64_nvme: crc64_k::NVME_PMULL_SMALL, #[cfg(feature = "crc64")] crc64_nvme_name: "aarch64/pmull-small", }; @@ -1123,31 +1079,31 @@ mod aarch64_tables { #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] const G3_S: KernelSet = KernelSet { #[cfg(feature = "crc16")] - crc16_ccitt: crc16_k::CCITT_PMULL[0], // pmull @ 11.97 GiB/s (G3: pmull beats pmull-small) + crc16_ccitt: crc16_k::CCITT_PMULL[0], #[cfg(feature = "crc16")] crc16_ccitt_name: "aarch64/pmull", #[cfg(feature = "crc16")] - crc16_ibm: crc16_k::IBM_PMULL[0], // pmull @ 12.01 GiB/s (G3: pmull beats pmull-small) + crc16_ibm: crc16_k::IBM_PMULL[0], #[cfg(feature = "crc16")] crc16_ibm_name: "aarch64/pmull", #[cfg(feature = "crc24")] - crc24_openpgp: crc24_k::OPENPGP_PMULL[0], // pmull @ 14.12 GiB/s + crc24_openpgp: crc24_k::OPENPGP_PMULL[0], #[cfg(feature = "crc24")] crc24_openpgp_name: "aarch64/pmull", #[cfg(feature = "crc32")] - crc32_ieee: crc32_k::CRC32_PMULL_SMALL_KERNEL, // pmull-small @ 8.66 GiB/s + crc32_ieee: crc32_k::CRC32_PMULL_SMALL_KERNEL, #[cfg(feature = "crc32")] crc32_ieee_name: "aarch64/pmull-small", #[cfg(feature = "crc32")] - crc32c: crc32_k::CRC32C_PMULL_SMALL_KERNEL, // pmull-small @ 8.79 GiB/s + crc32c: crc32_k::CRC32C_PMULL_SMALL_KERNEL, #[cfg(feature = "crc32")] crc32c_name: "aarch64/pmull-small", #[cfg(feature = "crc64")] - crc64_xz: crc64_k::XZ_PMULL[0], // pmull @ 18.05 GiB/s + crc64_xz: crc64_k::XZ_PMULL[0], #[cfg(feature = "crc64")] crc64_xz_name: "aarch64/pmull", #[cfg(feature = "crc64")] - crc64_nvme: crc64_k::NVME_PMULL[0], // pmull @ 18.01 GiB/s + crc64_nvme: crc64_k::NVME_PMULL[0], #[cfg(feature = "crc64")] crc64_nvme_name: "aarch64/pmull", }; @@ -1163,23 +1119,23 @@ mod aarch64_tables { #[cfg(feature = "crc16")] crc16_ibm_name: "aarch64/pmull-eor3", #[cfg(feature = "crc24")] - crc24_openpgp: crc24_k::OPENPGP_PMULL[0], // pmull @ 32.78 GiB/s + crc24_openpgp: crc24_k::OPENPGP_PMULL[0], #[cfg(feature = "crc24")] crc24_openpgp_name: "aarch64/pmull", #[cfg(feature = "crc32")] - crc32_ieee: crc32_k::CRC32_PMULL_SMALL_KERNEL, // pmull-small @ 31.76 GiB/s + crc32_ieee: crc32_k::CRC32_PMULL_SMALL_KERNEL, #[cfg(feature = "crc32")] crc32_ieee_name: "aarch64/pmull-small", #[cfg(feature = "crc32")] - crc32c: crc32_k::CRC32C_PMULL_SMALL_KERNEL, // pmull-small @ 32.26 GiB/s + crc32c: crc32_k::CRC32C_PMULL_SMALL_KERNEL, #[cfg(feature = "crc32")] crc32c_name: "aarch64/pmull-small", #[cfg(feature = "crc64")] - crc64_xz: crc64_k::XZ_PMULL[0], // pmull @ 35.34 GiB/s + crc64_xz: crc64_k::XZ_PMULL[0], #[cfg(feature = "crc64")] crc64_xz_name: "aarch64/pmull", #[cfg(feature = "crc64")] - crc64_nvme: crc64_k::NVME_PMULL[1], // pmull-2way @ 34.60 GiB/s (G3: 2way beats 1way) + crc64_nvme: crc64_k::NVME_PMULL[1], #[cfg(feature = "crc64")] crc64_nvme_name: "aarch64/pmull-2way", }; @@ -1195,23 +1151,23 @@ mod aarch64_tables { #[cfg(feature = "crc16")] crc16_ibm_name: "aarch64/pmull-eor3-g3-ibm-hybrid", #[cfg(feature = "crc24")] - crc24_openpgp: crc24_k::OPENPGP_PMULL[0], // pmull @ 37.90 GiB/s + crc24_openpgp: crc24_k::OPENPGP_PMULL[0], #[cfg(feature = "crc24")] crc24_openpgp_name: "aarch64/pmull", #[cfg(feature = "crc32")] - crc32_ieee: crc32_k::CRC32_PMULL_EOR3[0], // pmull-eor3-v9s3x2e-s3 @ 46.29 GiB/s + crc32_ieee: crc32_k::CRC32_PMULL_EOR3[0], #[cfg(feature = "crc32")] crc32_ieee_name: "aarch64/pmull-eor3-v9s3x2e-s3", #[cfg(feature = "crc32")] - crc32c: crc32_k::CRC32C_PMULL_EOR3[0], // pmull-eor3-v9s3x2e-s3 @ 46.08 GiB/s + crc32c: crc32_k::CRC32C_PMULL_EOR3[0], #[cfg(feature = "crc32")] crc32c_name: "aarch64/pmull-eor3-v9s3x2e-s3", #[cfg(feature = "crc64")] - crc64_xz: crc64_k::XZ_PMULL_EOR3[0], // pmull-eor3 @ 38.04 GiB/s + crc64_xz: crc64_k::XZ_PMULL_EOR3[0], #[cfg(feature = "crc64")] crc64_xz_name: "aarch64/pmull-eor3", #[cfg(feature = "crc64")] - crc64_nvme: crc64_k::NVME_PMULL_EOR3[0], // pmull-eor3 @ 38.05 GiB/s (G3: eor3 beats pmull) + crc64_nvme: crc64_k::NVME_PMULL_EOR3[0], #[cfg(feature = "crc64")] crc64_nvme_name: "aarch64/pmull-eor3", }; @@ -1677,8 +1633,8 @@ mod x86_64_tables { /// Zen4 CRC64/XZ large-path hybrid. /// - /// 2-way is strongest around lower "large" sizes, while 8-way closes the - /// remaining gap at xl-scale buffers. + /// Uses the 2-way kernel below the table threshold and the 8-way kernel + /// above it. #[cfg(feature = "crc64")] #[inline] fn zen4_crc64_xz_l_hybrid(crc: u64, data: &[u8]) -> u64 { @@ -1689,28 +1645,9 @@ mod x86_64_tables { } } - // Zen5 Table + // Zen4 table. // - // Benchmark source: `src/checksum/bench_baseline/linux_x86-64_zen5_kernels.txt` // Features: VPCLMULQDQ + AVX-512 - // - // Key differences vs Zen4: - // - Different optimal multi-stream counts for CRC16/24 kernels - // - CRC64 (large) prefers VPCLMUL-2way over the 4×512 kernel - // Zen4 Table - // - // Benchmark source: Namespace linux-x86 runner (2026-01-20) - // Features: VPCLMULQDQ + AVX-512 - // Peak throughputs: CRC-16 ~80 GiB/s, CRC-32 ~78 GiB/s, CRC-64 ~75 GiB/s - // - // Optimal kernels per (variant, peak): - // crc16/ccitt: vpclmul, streams=4, 79.87 GiB/s - // crc16/ibm: vpclmul, streams=4, 77.96 GiB/s - // crc24/openpgp: vpclmul, streams=7, 42.96 GiB/s - // crc32/ieee: vpclmul, streams=2, 78.29 GiB/s - // crc32c: fusion-vpclmul-v3x2, streams=1, 72.53 GiB/s - // crc64/xz: vpclmul, streams=2, 71.56 GiB/s - // crc64/nvme: vpclmul, streams=2, 75.18 GiB/s pub static ZEN4_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::x86::VPCLMUL_READY .union(crate::platform::caps::x86::PCLMUL_READY) @@ -1731,11 +1668,11 @@ mod x86_64_tables { #[cfg(feature = "crc24")] crc24_openpgp_name: "x86_64/pclmul-small", #[cfg(feature = "crc32")] - crc32_ieee: crc32_k::CRC32_PCLMUL_SMALL_KERNEL, // pclmul-small @ 9.30 GiB/s + crc32_ieee: crc32_k::CRC32_PCLMUL_SMALL_KERNEL, #[cfg(feature = "crc32")] crc32_ieee_name: "x86_64/pclmul-small", #[cfg(feature = "crc32")] - crc32c: crc32_k::CRC32C_HWCRC[0], // hwcrc @ 27.73 GiB/s + crc32c: crc32_k::CRC32C_HWCRC[0], #[cfg(feature = "crc32")] crc32c_name: "x86_64/hwcrc", #[cfg(feature = "crc64")] @@ -1750,81 +1687,81 @@ mod x86_64_tables { s: KernelSet { #[cfg(feature = "crc16")] - crc16_ccitt: crc16_k::CCITT_VPCLMUL[1], // 2-way @ 18.12 GiB/s + crc16_ccitt: crc16_k::CCITT_VPCLMUL[1], #[cfg(feature = "crc16")] crc16_ccitt_name: "x86_64/vpclmul-2way", #[cfg(feature = "crc16")] - crc16_ibm: crc16_k::IBM_VPCLMUL[1], // 2-way @ 19.90 GiB/s + crc16_ibm: crc16_k::IBM_VPCLMUL[1], #[cfg(feature = "crc16")] crc16_ibm_name: "x86_64/vpclmul-2way", #[cfg(feature = "crc24")] - crc24_openpgp: crc24_k::OPENPGP_VPCLMUL[3], // 7-way @ 17.28 GiB/s + crc24_openpgp: crc24_k::OPENPGP_VPCLMUL[3], #[cfg(feature = "crc24")] crc24_openpgp_name: "x86_64/vpclmul-7way", #[cfg(feature = "crc32")] - crc32_ieee: crc32_k::CRC32_VPCLMUL[0], // 1-way @ 19.15 GiB/s + crc32_ieee: crc32_k::CRC32_VPCLMUL[0], #[cfg(feature = "crc32")] crc32_ieee_name: "x86_64/vpclmul", #[cfg(feature = "crc32")] - crc32c: crc32_k::CRC32C_HWCRC[0], // hwcrc @ 23.59 GiB/s + crc32c: crc32_k::CRC32C_HWCRC[0], #[cfg(feature = "crc32")] crc32c_name: "x86_64/hwcrc", #[cfg(feature = "crc64")] - crc64_xz: crc64_k::XZ_VPCLMUL[0], // 1-way @ 19.48 GiB/s + crc64_xz: crc64_k::XZ_VPCLMUL[0], #[cfg(feature = "crc64")] crc64_xz_name: "x86_64/vpclmul", #[cfg(feature = "crc64")] - crc64_nvme: crc64_k::NVME_VPCLMUL[0], // 1-way @ 20.70 GiB/s + crc64_nvme: crc64_k::NVME_VPCLMUL[0], #[cfg(feature = "crc64")] crc64_nvme_name: "x86_64/vpclmul", }, m: KernelSet { #[cfg(feature = "crc16")] - crc16_ccitt: crc16_k::CCITT_VPCLMUL[1], // 2-way @ 61.24 GiB/s + crc16_ccitt: crc16_k::CCITT_VPCLMUL[1], #[cfg(feature = "crc16")] crc16_ccitt_name: "x86_64/vpclmul-2way", #[cfg(feature = "crc16")] - crc16_ibm: crc16_k::IBM_VPCLMUL[1], // 2-way @ 64.84 GiB/s + crc16_ibm: crc16_k::IBM_VPCLMUL[1], #[cfg(feature = "crc16")] crc16_ibm_name: "x86_64/vpclmul-2way", #[cfg(feature = "crc24")] - crc24_openpgp: crc24_k::OPENPGP_VPCLMUL[2], // 4-way @ 34.72 GiB/s (bench: 4way beats 8way) + crc24_openpgp: crc24_k::OPENPGP_VPCLMUL[2], #[cfg(feature = "crc24")] crc24_openpgp_name: "x86_64/vpclmul-4way", #[cfg(feature = "crc32")] - crc32_ieee: crc32_k::CRC32_VPCLMUL[1], // 2-way @ 64.13 GiB/s + crc32_ieee: crc32_k::CRC32_VPCLMUL[1], #[cfg(feature = "crc32")] crc32_ieee_name: "x86_64/vpclmul-2way", #[cfg(feature = "crc32")] - crc32c: crc32_k::CRC32C_FUSION_VPCLMUL[0], // fusion-vpclmul-v3x2 @ 58.96 GiB/s + crc32c: crc32_k::CRC32C_FUSION_VPCLMUL[0], #[cfg(feature = "crc32")] crc32c_name: "x86_64/fusion-vpclmul-v3x2", #[cfg(feature = "crc64")] - crc64_xz: crc64_k::XZ_VPCLMUL[1], // 2-way @ 66.51 GiB/s + crc64_xz: crc64_k::XZ_VPCLMUL[1], #[cfg(feature = "crc64")] crc64_xz_name: "x86_64/vpclmul-2way", #[cfg(feature = "crc64")] - crc64_nvme: crc64_k::NVME_VPCLMUL[1], // 2-way @ 66.51 GiB/s + crc64_nvme: crc64_k::NVME_VPCLMUL[1], #[cfg(feature = "crc64")] crc64_nvme_name: "x86_64/vpclmul-2way", }, l: KernelSet { #[cfg(feature = "crc16")] - crc16_ccitt: crc16_k::CCITT_VPCLMUL[2], // 4-way @ 73.97 GiB/s + crc16_ccitt: crc16_k::CCITT_VPCLMUL[2], #[cfg(feature = "crc16")] crc16_ccitt_name: "x86_64/vpclmul-4way", #[cfg(feature = "crc16")] - crc16_ibm: crc16_k::IBM_VPCLMUL[1], // 2-way @ 78.09 GiB/s + crc16_ibm: crc16_k::IBM_VPCLMUL[1], #[cfg(feature = "crc16")] crc16_ibm_name: "x86_64/vpclmul-2way", #[cfg(feature = "crc24")] - crc24_openpgp: crc24_k::OPENPGP_VPCLMUL[2], // 4-way @ 71.94 GiB/s (bench: 4way beats 2way) + crc24_openpgp: crc24_k::OPENPGP_VPCLMUL[2], #[cfg(feature = "crc24")] crc24_openpgp_name: "x86_64/vpclmul-4way", #[cfg(feature = "crc32")] - crc32_ieee: crc32_k::CRC32_VPCLMUL[1], // 2-way @ 72.57 GiB/s + crc32_ieee: crc32_k::CRC32_VPCLMUL[1], #[cfg(feature = "crc32")] crc32_ieee_name: "x86_64/vpclmul-2way", #[cfg(feature = "crc32")] @@ -1982,18 +1919,7 @@ mod x86_64_tables { // Generic x86-64 PCLMUL Table (conservative) // - // Benchmark source: historical Windows "Default" baseline (no AVX-512). - // Note: Windows benchmark baselines are no longer tracked in-tree. // Features: PCLMULQDQ only - // - // Optimal kernels per (variant, size_class): - // crc16/ccitt: xs=pclmul-small, s=pclmul, m=pclmul-7way, l=pclmul - // crc16/ibm: xs=pclmul-small, s=pclmul-4way, m=pclmul, l=pclmul-2way - // crc24/openpgp: xs=pclmul-small, s=pclmul, m=pclmul-2way, l=pclmul-2way - // crc32/ieee: xs=pclmul-small, s=pclmul-4way, m=pclmul, l=pclmul-2way - // crc32c: xs=hwcrc, s=hwcrc, m=hwcrc-2way, l=fusion-sse-v4s3x3-2way - // crc64/xz: xs=pclmul-small, s=pclmul-small, m=pclmul-4way, l=pclmul - // crc64/nvme: xs=pclmul-small, s=pclmul-small, m=pclmul-2way, l=pclmul-2way pub static GENERIC_X86_PCLMUL_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::x86::PCLMUL_READY.union(crate::platform::caps::x86::CRC32C_READY), boundaries: [64, 256, 4096], @@ -3039,13 +2965,8 @@ mod riscv64_tables { #[cfg(feature = "crc64")] // Keep CRC64 auto on portable slice-by-16 for now. // - // The repo already learned this lesson once on in-order RISC-V cores: the - // dedicated Zbc folds can look attractive on paper but still lose badly to - // slice16 once the dependency chains and merge overhead hit real hardware. - // - // The current RISE benchmark data still shows broad CRC64 losses across - // 32B..1MiB, so until we add the planned RISE hwprobe snapshot / per-runner - // tuning proof, the release-grade auto policy should stay conservative. + // No tracked target evidence currently justifies selecting the accelerated + // CRC64 kernels by default. #[cfg(feature = "crc64")] pub static RISCV64_CRC64_ZBC_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::riscv::ZBC, diff --git a/src/checksum/macros.rs b/src/checksum/macros.rs index 8eb4b832..7737a4f7 100644 --- a/src/checksum/macros.rs +++ b/src/checksum/macros.rs @@ -3,7 +3,7 @@ //! These macros eliminate boilerplate when defining buffered CRC wrappers //! and vectored (multi-buffer) dispatch. -/// Run a CRC kernel across multiple buffers, re-selecting the optimal kernel +/// Run a CRC kernel across multiple buffers, re-selecting the active kernel /// when the buffer size class changes. /// /// This is the shared core of every `*_vectored` / `*_io_slices` API and the @@ -152,7 +152,6 @@ macro_rules! define_crc_dispatch { /// - The struct definition with `inner`, `buffer`, and `len` fields /// - `new()`, `update()`, `finalize()`, and `reset()` methods /// - `Default` trait implementation -/// - `Drop` implementation that zeroizes the buffer via [`crate::traits::ct::zeroize`] /// /// # Arguments /// @@ -283,13 +282,5 @@ macro_rules! define_buffered_crc { } } - impl Drop for $name { - fn drop(&mut self) { - $crate::traits::ct::zeroize(&mut self.buffer[..]); - // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.len, 0) }; - core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); - } - } }; } diff --git a/src/checksum/mod.rs b/src/checksum/mod.rs index 85e6418f..b0f1c2c3 100644 --- a/src/checksum/mod.rs +++ b/src/checksum/mod.rs @@ -47,10 +47,10 @@ //! ```toml //! [dependencies] //! # Smallest CRC-32-only build -//! rscrypto = { version = "0.5.0", default-features = false, features = ["crc32"] } +//! rscrypto = { version = "0.7", default-features = false, features = ["crc32"] } //! //! # All checksum families -//! rscrypto = { version = "0.5.0", default-features = false, features = ["checksums"] } +//! rscrypto = { version = "0.7", default-features = false, features = ["checksums"] } //! ``` //! //! # API Conventions @@ -76,7 +76,7 @@ //! //! ```toml //! [dependencies] -//! rscrypto = { version = "0.5.0", default-features = false, features = ["crc32"] } +//! rscrypto = { version = "0.7", default-features = false, features = ["crc32"] } //! ``` mod common; diff --git a/src/hashes/common/mod.rs b/src/hashes/common/mod.rs index 71ed7adc..2875ebe6 100644 --- a/src/hashes/common/mod.rs +++ b/src/hashes/common/mod.rs @@ -1,17 +1,6 @@ -//! Common utilities for hash computation. +//! Shared hash implementation utilities. //! -//! This module provides: -//! - Cross-architecture SIMD abstractions for hash primitives -//! - Generic kernel selection and dispatch infrastructure -//! - Software prefetch helpers for optimal memory access patterns -//! - Shared compression patterns for hash algorithms -//! -//! # Design Philosophy -//! -//! The patterns here mirror `checksum::common` to maintain consistency -//! across the rscrypto crate ecosystem. Hash algorithms share many -//! optimization techniques (SIMD vectorization, prefetch hints, etc.) -//! that can be centralized here. +//! This module currently owns the AArch64 BLAKE3 prefetch helper. #[cfg(all(feature = "blake3", target_arch = "aarch64"))] pub mod prefetch; diff --git a/src/hashes/common/prefetch.rs b/src/hashes/common/prefetch.rs index 590c2632..83952c65 100644 --- a/src/hashes/common/prefetch.rs +++ b/src/hashes/common/prefetch.rs @@ -1,6 +1,6 @@ //! Software prefetch helpers for hash algorithm kernels. //! -//! Provides platform-specific `prefetch_read_l1` for optimal memory access +//! Provides the platform-specific `prefetch_read_l1` hint //! patterns in multi-chunk hash computation (e.g., Blake3 NEON hot loops). //! //! Prefetch instructions are CPU hints — invalid addresses are silently ignored. diff --git a/src/hashes/crypto/ascon/dispatch_tables.rs b/src/hashes/crypto/ascon/dispatch_tables.rs index b42959d8..2e507958 100644 --- a/src/hashes/crypto/ascon/dispatch_tables.rs +++ b/src/hashes/crypto/ascon/dispatch_tables.rs @@ -45,11 +45,9 @@ pub static DEFAULT_TABLE: DispatchTable = DispatchTable { #[cfg(target_arch = "aarch64")] pub static AARCH64_NEON_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, - // Scalar is faster than NEON for single-state Ascon: the 320-bit state - // (5 × u64) fits in 5 GPRs with native 1-cycle `ROR`, while NEON has no - // 64-bit vector rotate and must simulate each with SHR+SHL+OR (3 ops). - // The duplicated-lane single-state kernel doubles work for no benefit. - // NEON x2 batch path (used by `digest_many`) is wired separately. + // The single-state policy stays scalar: duplicating each of the five state + // words across NEON lanes does not add independent work. The NEON x2 batch + // path is wired separately. xs: KernelId::Portable, s: KernelId::Portable, m: KernelId::Portable, @@ -59,11 +57,9 @@ pub static AARCH64_NEON_TABLE: DispatchTable = DispatchTable { #[cfg(target_arch = "x86_64")] pub static X86_AVX2_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, - // Scalar is faster than AVX2 for single-state Ascon: the 320-bit state - // (5 × u64) fits in 5 GPRs with native 1-cycle `ROR`, while the - // duplicated-lane AVX2 kernel broadcasts each word across 4 lanes (4× - // the work). The broadcast + extract overhead dominates for a single - // state. AVX2 x4 batch path (used by `digest_many`) is wired separately. + // The single-state policy stays scalar: broadcasting each state word across + // four AVX2 lanes does not add independent work. The AVX2 x4 batch path is + // wired separately. xs: KernelId::Portable, s: KernelId::Portable, m: KernelId::Portable, @@ -73,12 +69,9 @@ pub static X86_AVX2_TABLE: DispatchTable = DispatchTable { #[cfg(target_arch = "x86_64")] pub static X86_AVX512_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, - // Scalar is faster than AVX-512 for single-state Ascon: the 320-bit - // state (5 × u64) fits in 5 GPRs with native 1-cycle `ROR`, while the - // duplicated-lane AVX-512 kernel broadcasts each word across 8 lanes - // (8× the work). Even with VPTERNLOGQ and VPROLQ, the broadcast + - // extract overhead dominates for a single state. AVX-512 x8 batch path - // (used by `digest_many`) is wired separately. + // The single-state policy stays scalar: broadcasting each state word across + // eight AVX-512 lanes does not add independent work. The AVX-512 x8 batch + // path is wired separately. xs: KernelId::Portable, s: KernelId::Portable, m: KernelId::Portable, diff --git a/src/hashes/crypto/blake3/dispatch.rs b/src/hashes/crypto/blake3/dispatch.rs index 72315088..d64cfd0e 100644 --- a/src/hashes/crypto/blake3/dispatch.rs +++ b/src/hashes/crypto/blake3/dispatch.rs @@ -22,24 +22,16 @@ use crate::{backend::cache::OnceCache, hashes::crypto::dispatch_util::SizeClassD #[inline] #[must_use] fn allow_avx2_hash_many_one_chunk_fast_path(caps: Caps) -> bool { - caps.has(x86::AVX512_READY) - && !caps.has(x86::AMX_TILE) - && !caps.has(x86::AMX_INT8) - && !caps.has(x86::AMX_BF16) - && !caps.has(x86::AMX_FP16) - && !caps.has(x86::AMX_COMPLEX) + caps.has(x86::AVX512_READY) && !caps.has(x86::INTEL_SAPPHIRE_RAPIDS) } -/// True on wide-pipeline CPUs (Zen 5+, Intel non-AMX) where hash_many's SIMD -/// setup cost is not amortized at ≤ 4 blocks. On narrow-pipeline AMD (Zen 4), -/// hash_many is beneficial even at 4 blocks. +/// Return the configured four-block policy class for x86-64. #[cfg(target_arch = "x86_64")] #[inline] #[must_use] fn is_wide_pipeline_for_hash_many(caps: Caps) -> bool { - // Zen 5+: 6-wide dispatch, hash_many overhead > sequential at 4 blocks. - // Intel ICL-class (AVX-512, no AMX): same pattern. - // Zen 4 (AMD, not Zen5): 4-wide, hash_many amortizes at 4 blocks. + // Zen 5 and Intel AVX-512 use the wide-pipeline policy; earlier AMD uses + // the alternate four-block policy. if caps.has(x86::AMD) { caps.has(x86::AMD_ZEN5) } else { @@ -374,3 +366,24 @@ pub(crate) fn hash_many_wide_pipeline() -> bool { pub(crate) fn avx2_available() -> bool { resolved().avx2_available } + +#[cfg(all(test, target_arch = "x86_64"))] +mod tests { + use super::*; + + const ALL_AMX: Caps = x86::AMX_TILE + .union(x86::AMX_BF16) + .union(x86::AMX_INT8) + .union(x86::AMX_FP16) + .union(x86::AMX_COMPLEX); + + #[test] + fn sapphire_rapids_shortcut_policy_does_not_depend_on_amx_permission() { + let sapphire_rapids = x86::AVX512_READY | x86::INTEL_SAPPHIRE_RAPIDS; + assert!(!allow_avx2_hash_many_one_chunk_fast_path(sapphire_rapids)); + assert!(!allow_avx2_hash_many_one_chunk_fast_path(sapphire_rapids | ALL_AMX)); + + assert!(allow_avx2_hash_many_one_chunk_fast_path(x86::AVX512_READY)); + assert!(allow_avx2_hash_many_one_chunk_fast_path(x86::AVX512_READY | ALL_AMX)); + } +} diff --git a/src/hashes/crypto/blake3/dispatch_tables.rs b/src/hashes/crypto/blake3/dispatch_tables.rs index 2d74ff72..75919a20 100644 --- a/src/hashes/crypto/blake3/dispatch_tables.rs +++ b/src/hashes/crypto/blake3/dispatch_tables.rs @@ -486,14 +486,13 @@ pub static PROFILE_DEFAULT_KIND: FamilyProfile = default_kind_profile(); #[cfg(not(target_arch = "x86_64"))] pub static PROFILE_PORTABLE: FamilyProfile = portable_profile(); -// Family Profile: X86_AVX512_AMX +// Family Profile: INTEL_SAPPHIRE_RAPIDS #[cfg(target_arch = "x86_64")] -pub static PROFILE_X86_AVX512_AMX: FamilyProfile = FamilyProfile { +pub static PROFILE_INTEL_SAPPHIRE_RAPIDS: FamilyProfile = FamilyProfile { dispatch: DispatchTable { boundaries: [64, 1024, 4096], - // AVX-512 compress uses native `vprord` (1 µop) vs SSE4.1/AVX2 shift+or - // emulation (3 instructions). For single-block compress in the one-chunk - // path, AVX-512 is strictly faster regardless of input size. + // This profile uses AVX-512 for every size class; its selector has no + // single-block crossover to another x86 kernel. xs: KernelId::X86Avx512, s: KernelId::X86Avx512, m: KernelId::X86Avx512, @@ -534,9 +533,8 @@ pub static PROFILE_X86_AVX512_AMX: FamilyProfile = FamilyProfile { pub static PROFILE_X86_AVX512: FamilyProfile = FamilyProfile { dispatch: DispatchTable { boundaries: [64, 1024, 4096], - // AVX-512 compress uses native `vprord` (1 µop) vs SSE4.1/AVX2 shift+or - // emulation (3 instructions). For single-block compress in the one-chunk - // path, AVX-512 is strictly faster regardless of input size. + // This profile uses AVX-512 for every size class; its selector has no + // single-block crossover to another x86 kernel. xs: KernelId::X86Avx512, s: KernelId::X86Avx512, m: KernelId::X86Avx512, @@ -762,25 +760,14 @@ pub static PROFILE_POWER10: FamilyProfile = FamilyProfile { }, }; -#[inline] -#[must_use] -#[cfg(target_arch = "x86_64")] -fn has_any_amx(caps: Caps) -> bool { - caps.has(x86::AMX_TILE) - || caps.has(x86::AMX_INT8) - || caps.has(x86::AMX_BF16) - || caps.has(x86::AMX_FP16) - || caps.has(x86::AMX_COMPLEX) -} - #[inline] #[must_use] pub fn select_profile_for_caps(caps: Caps) -> &'static FamilyProfile { #[cfg(target_arch = "x86_64")] { if caps.has(x86::AVX512_READY) { - return if has_any_amx(caps) { - &PROFILE_X86_AVX512_AMX + return if caps.has(x86::INTEL_SAPPHIRE_RAPIDS) { + &PROFILE_INTEL_SAPPHIRE_RAPIDS } else { &PROFILE_X86_AVX512 }; @@ -873,3 +860,35 @@ pub fn select_parallel_table_for_caps(caps: Caps) -> &'static ParallelTable { pub fn select_streaming_parallel_table_for_caps(caps: Caps) -> &'static ParallelTable { &select_profile_for_caps(caps).streaming_parallel } + +#[cfg(all(test, target_arch = "x86_64"))] +mod tests { + use super::*; + + const ALL_AMX: Caps = x86::AMX_TILE + .union(x86::AMX_BF16) + .union(x86::AMX_INT8) + .union(x86::AMX_FP16) + .union(x86::AMX_COMPLEX); + + #[test] + fn profile_selection_uses_processor_identity_not_amx_permission() { + let sapphire_rapids = x86::AVX512_READY | x86::INTEL_SAPPHIRE_RAPIDS; + assert!(core::ptr::eq( + select_profile_for_caps(sapphire_rapids), + &PROFILE_INTEL_SAPPHIRE_RAPIDS + )); + assert!(core::ptr::eq( + select_profile_for_caps(sapphire_rapids | ALL_AMX), + &PROFILE_INTEL_SAPPHIRE_RAPIDS + )); + assert!(core::ptr::eq( + select_profile_for_caps(x86::AVX512_READY), + &PROFILE_X86_AVX512 + )); + assert!(core::ptr::eq( + select_profile_for_caps(x86::AVX512_READY | ALL_AMX), + &PROFILE_X86_AVX512 + )); + } +} diff --git a/src/hashes/crypto/blake3/kernels.rs b/src/hashes/crypto/blake3/kernels.rs index 362df558..eb68b377 100644 --- a/src/hashes/crypto/blake3/kernels.rs +++ b/src/hashes/crypto/blake3/kernels.rs @@ -614,10 +614,8 @@ fn root_output_block_words_inline( #[cfg(target_arch = "aarch64")] if id == Blake3KernelId::Aarch64Neon { - // Use portable compress for single-block XOF emit: NEON register load/store - // overhead is not worth it for a single 64-byte compress. The official blake3 - // crate makes the same choice (portable::compress_xof for NEON). - // Bulk emit (root_output_blocks4_neon) still uses NEON. + // Single-block XOF emit follows upstream BLAKE3's portable NEON policy. + // Bulk emit remains on root_output_blocks4_neon. let words = super::compress(chaining_value, block_words, counter, block_len, flags); write_root_output_words(out, &words); return; @@ -664,8 +662,7 @@ fn root_output_block_bytes_inline( { match id { Blake3KernelId::X86Avx512 => { - // For AVX-512 dispatch, the word-form path is faster than the SSE4.1 - // byte helper even after decoding this one block. + // The AVX-512 policy uses the word-form path after decoding this block. let block_words = super::words16_from_le_bytes_64(block_bytes); root_output_block_words_inline(id, chaining_value, &block_words, counter, block_len, flags, out); return; @@ -1481,9 +1478,8 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( // 2. `children` has exactly four OUT_LEN-byte child CVs, so `parents` points to two packed 64-byte // parent blocks. // 3. `out` has exactly two contiguous OUT_LEN-byte parent CV slots. - // 4. The direct two-parent pair route was measured faster than the AVX2 assembly tail on Sapphire - // Rapids. Routing it through the generic reducer was slower, so this path is intentionally - // exact. + // 4. The manually maintained AVX2 tail policy routes exactly two parents through the owned pair + // kernel. unsafe { super::x86_64::avx2::parent_cv2_owned(&parents, &key_words, flags, out[0].as_mut_ptr()); } @@ -1500,8 +1496,8 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( // blocks for the pair reducer, and `children[4]` starts the final packed parent block. // 3. `out[0..2]` is contiguous for the pair output, and `out[2]` is writable for the final serial // parent CV. - // 4. The direct pair-plus-serial route was measured faster than the AVX2 assembly tail on Sapphire - // Rapids. Larger reductions keep 2/3 remainders on assembly until separately measured. + // 4. The manually maintained AVX2 tail policy routes exactly three parents through one pair plus + // one serial parent. unsafe { super::x86_64::avx2::parent_cv2_owned(&parents, &key_words, flags, out[0].as_mut_ptr()); parent_one_avx2_owned_serial_from_block( @@ -1525,8 +1521,7 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( // 1. Dispatch selected the AVX2 kernel, so AVX2 is available. // 2. `ptrs` points to four readable packed 64-byte parent blocks. // 3. `out` is writable for four contiguous OUT_LEN-byte parent CV outputs. - // 4. Direct output avoids the generic reducer's temporary copy and measured faster on Sapphire - // Rapids. + // 4. Direct output avoids the generic reducer's temporary copy. unsafe { super::x86_64::asm::hash_many_avx2( ptrs.as_ptr(), @@ -1554,8 +1549,7 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( // 1. Dispatch selected the AVX2 kernel, so AVX2 is available. // 2. `ptrs` contains 8 readable packed 64-byte parent block pointers. // 3. `out` is writable for 8 contiguous OUT_LEN-byte parent CV outputs. - // 4. Direct output avoids the generic reducer's temporary copy and measured faster on Sapphire - // Rapids. + // 4. Direct output avoids the generic reducer's temporary copy. unsafe { super::x86_64::avx2::hash8_owned(&ptrs, 1, &key_words, 0, false, parent_flags, 0, 0, out[0].as_mut_ptr()); } @@ -1576,8 +1570,7 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( // 1. Dispatch selected the AVX2 kernel, so AVX2 is available. // 2. `ptrs[0]` points to one packed 64-byte parent block. // 3. `tmp[0]` is writable for one OUT_LEN-byte parent CV. - // 4. The one-parent serial route was measured faster than the AVX2 assembly tail on Sapphire - // Rapids. + // 4. The manually maintained AVX2 tail policy routes one parent through the serial owned kernel. unsafe { parent_one_avx2_owned_serial_from_block(ptrs[0], key_words, flags, &mut tmp[0]); } @@ -2310,10 +2303,8 @@ unsafe fn chunk_compress_blocks_s390x_vector( ) { debug_assert_eq!(blocks.len() % BLOCK_LEN, 0); - // On IBM Z, the vector per-block wrapper has measurable fixed overhead on - // short one-chunk bodies (e.g. 256B/1024B inputs). Route those - // small batches through the portable scalar loop and keep the vector path - // for larger batches where it clearly wins. + // The IBM Z policy routes incomplete one-chunk bodies through the portable + // loop and reserves the vector wrapper for a complete chunk. const SHORT_BATCH_PORTABLE_MAX_BLOCKS: usize = (CHUNK_LEN / BLOCK_LEN) - 1; // 15 let num_blocks = blocks.len() / BLOCK_LEN; if num_blocks <= SHORT_BATCH_PORTABLE_MAX_BLOCKS { @@ -3403,7 +3394,7 @@ unsafe fn hash_many_contiguous_avx2_inner( // 1. Dispatch selects this wrapper only after AVX2 support is available. // 2. `ptrs` point to `DEGREE` in-bounds full-chunk inputs. // 3. `out` is valid for `DEGREE * OUT_LEN` bytes. - // 4. Sub-degree tails are handled below by measured per-degree routing. + // 4. Sub-degree tails are handled below by the explicit per-degree policy. unsafe { super::x86_64::avx2::hash8_owned( &ptrs, @@ -3454,8 +3445,7 @@ unsafe fn hash_many_contiguous_avx2_inner( // 1. This wrapper is only selected after AVX2 dispatch. // 2. `input` is readable for one full chunk. // 3. `out` is writable for one OUT_LEN-byte CV. - // 4. The one-chunk tail was measured faster than the remaining AVX2 assembly tail on Sapphire - // Rapids. + // 4. The manually maintained AVX2 tail policy routes one chunk through the serial owned kernel. unsafe { hash_one_chunk_avx2_owned_serial(input, key, counter, flags, out); } @@ -3466,20 +3456,20 @@ unsafe fn hash_many_contiguous_avx2_inner( // 1. This wrapper is only selected after AVX2 dispatch. // 2. `input` is readable for two full chunks. // 3. `out` is writable for two OUT_LEN-byte CVs. - // 4. The two-chunk pair route was measured faster than the AVX2 assembly tail on Sapphire Rapids. + // 4. The manually maintained AVX2 tail policy routes two chunks through the owned pair kernel. unsafe { super::x86_64::avx2::hash2_chunks_owned(input, key, counter, flags, out); } return; } if matches!(num_chunks, 3 | 5..=7) { - // The owned duplicate-lane tail wins for 3/5/6/7 contiguous chunk tails - // on Sapphire Rapids. Keep 4 on assembly; it was measured faster. - // SAFETY: Routing this measured AVX2 tail because: + // The manually maintained tail policy uses duplicate lanes for + // 3/5/6/7 contiguous chunks and keeps four on assembly. + // SAFETY: Routing this shape-specific AVX2 tail because: // 1. This wrapper is only selected after AVX2 dispatch. // 2. `input` is readable for `num_chunks * CHUNK_LEN`. // 3. `out` is writable for `num_chunks * OUT_LEN`. - // 4. The `matches!` guard restricts `num_chunks` to sub-degree values that won in benchmarks. + // 4. The `matches!` guard restricts `num_chunks` to the configured duplicate-lane shapes. unsafe { hash_many_avx2_owned_duplicate_tail(input, num_chunks, key, counter, flags, out); } @@ -3488,7 +3478,7 @@ unsafe fn hash_many_contiguous_avx2_inner( // Use the upstream-grade AVX2 asm `hash_many` backend for the remaining // sub-degree tails. Passing `num_inputs = num_chunks` avoids wasting lanes - // for the 4 case where assembly is still faster. + // for the four-chunk shape assigned to assembly by the tail policy. // SAFETY: This wrapper is only selected when AVX2 is available (checked // by dispatch). `input` is valid for `num_chunks * CHUNK_LEN` bytes, so // each `input.add(i * CHUNK_LEN)` stays in-bounds. `out` is valid for @@ -3709,15 +3699,13 @@ unsafe fn hash_many_contiguous_avx512_wrapper( debug_assert!(num_chunks < super::x86_64::avx512::DEGREE); debug_assert!(flags <= u8::MAX as u32); // Use the AVX-512 asm `hash_many` backend for the sub-degree tail - // (1–15 chunks). The assembly has efficient internal cascade with lane - // masking — Rust-side cascades to SSE4.1/AVX2 were benchmarked and - // regressed 11–36% on all x86_64 platforms. + // (1–15 chunks). The assembly performs its internal cascade with lane + // masking. // // The exception is the 15-chunk contiguous tail on AVX512DQ CPUs: the - // owned duplicate-lane 16-way batch is faster than the assembly cascade - // there, while every smaller measured tail remains slower. + // owned duplicate-lane 16-way batch is the configured exception. if num_chunks == 15 && avx512_owned_hash_many_available() { - // SAFETY: Routing this measured AVX-512 tail because: + // SAFETY: Routing this shape-specific AVX-512 tail because: // 1. Dispatch selected the AVX-512 kernel. // 2. `avx512_owned_hash_many_available()` checked the additional AVX512DQ requirement. // 3. `input` is readable for `15 * CHUNK_LEN` bytes. diff --git a/src/hashes/crypto/blake3/mod.rs b/src/hashes/crypto/blake3/mod.rs index 9eddae90..e2844434 100644 --- a/src/hashes/crypto/blake3/mod.rs +++ b/src/hashes/crypto/blake3/mod.rs @@ -950,7 +950,7 @@ fn reduce_power_of_two_chunk_cvs_any( return cvs[0]; } if cvs.len() <= 16 { - // Small subtrees are faster to fold serially (and avoid allocation / thread coordination). + // The configured small-tree path avoids allocation and thread coordination. return reduce_power_of_two_chunk_cvs(kernel, key_words, flags, cvs); } @@ -972,15 +972,13 @@ fn reduce_power_of_two_chunk_cvs_any( let pairs = cur_len / 2; debug_assert!(pairs != 0); - // For large levels, parallelize parent folding. For small levels, the SIMD - // parent kernel is faster than coordinating threads. + // Parallelize only levels admitted by the explicit parent-fold threshold. // // Note: at 1 MiB (1024 chunks), the first reduction level is 512 pairs. If // we require a huge minimum here, we'd never parallelize parent folding on // common “large input” sizes. // - // 256 pairs = 512 child CVs. That's enough work (and memory traffic) to - // amortize Rayon scheduling overhead on typical desktop/server CPUs. + // The threshold is a manually maintained scheduling policy. const MIN_PAIRS_FOR_PARALLEL: usize = 256; // Helper: fold one reduction level, parallel (rayon) or serial. @@ -2067,8 +2065,8 @@ fn root_output_oneshot( } } - // Small exact trees are common in oneshot benchmarks (4KiB/16KiB). For these, - // bypass the generic CV-stack builder and reduce the leaves directly. + // Small exact trees can bypass the generic CV-stack builder and reduce the + // leaves directly. if remainder == 0 && full_chunks.is_power_of_two() { if full_chunks <= MAX_SIMD_DEGREE { #[cfg(target_endian = "little")] diff --git a/src/hashes/crypto/keccak.rs b/src/hashes/crypto/keccak.rs index 3556dac6..ec54dc6e 100644 --- a/src/hashes/crypto/keccak.rs +++ b/src/hashes/crypto/keccak.rs @@ -58,13 +58,11 @@ const RC: [u64; KECCAKF_ROUNDS] = [ // State stays as `&mut [u64; 25]` so LLVM generates uniform `[rsp + const]` // spill patterns. A 5-element buffer is reused for θ and χ. Serial ρ+π chain // with hardcoded PI/RHO indices. Max ~8 simultaneous live locals. -// Measured: +28% Zen5, +30% SPR, +9% Zen4/ICL vs the named-variable version. // // **aarch64 / register-rich targets (≥30 GPRs):** named-variable state. // All 25 lanes live in registers with good ILP. The 25 `b` temporaries for // ρ+π also fit without spilling. Avoids load/store traffic that the array // version introduces. -// Measured: 8% faster than array-based on Graviton4. /// x86-64 / s390x / generic: array-based Keccak-f[1600]. /// @@ -422,8 +420,8 @@ pub(crate) fn keccakf_portable(state: &mut [u64; 25]) { /// /// Loads `state[i] ^ block_lane_i` directly into named register variables for /// rate lanes, and `state[i]` for capacity lanes. This eliminates the -/// write-then-reload round-trip that separate `xor_block_into` + `keccakf_portable` -/// incurs (~34 memory ops saved per SHA3-256 block). +/// write-then-reload round-trip incurred by separate `xor_block_into` and +/// `keccakf_portable` calls. /// /// Since `RATE` is a const generic, `RATE / 8` is compile-time known and LLVM /// eliminates all `if lane < lanes` branches — the result is straight-line code. @@ -518,22 +516,16 @@ fn keccakf_absorb_portable(state: &mut [u64; 25], block: &[u8 // // Direct-call permuters replace the old function-pointer dispatch. Each // platform gets a concrete `Permuter` that calls the best kernel directly, -// allowing LLVM to inline the permutation into the absorb loop. This is the -// single most impactful change for SHA-3 throughput. +// allowing LLVM to inline the permutation into the absorb loop. // // - x86_64: `X86Permuter` → `keccakf_portable` for single-state, AVX-512VL two-state `permute_x2` -// when available. SIMD evaluation for single-state Keccak remains negative: -// * AVX-512 χ-only: 9-38% SLOWER on Zen4/5, ICL, SPR (GPR↔SIMD crossing > VPTERNLOG savings) -// * AVX2: worse than AVX-512 (no VPTERNLOG, 3 ops for χ vs 1) -// * BMI2: LLVM already emits RORX; ANDN saves <5 ops/round after lane-complementing chi -// * Full SIMD: 25 u64 lanes need 13+ YMM registers; θ/ρ/π have no efficient SIMD mapping -// The two-state path has different economics: each vector lane carries an -// independent public XOF stream, so VPROLQ/VPTERNLOGQ reduce the paired +// when available. A single state does not fill both lanes, while the paired path maps one +// independent public XOF stream to each lane. The two-state path has different economics: each +// vector lane carries an independent public XOF stream, so VPROLQ/VPTERNLOGQ reduce the paired // permutation work instead of duplicating one state. // - generic: `InlinePermuter` → `keccakf_portable`. -// - aarch64: `Aarch64Permuter` → portable for single-state (the 1-state SHA3 CE kernel is ~1.9× -// slower on Neoverse V1/V2). SHA3 CE is used only for the 2-state interleaved path -// (`digest_pair`). +// - aarch64: `Aarch64Permuter` uses the selected scalar single-state path and SHA3 CE for the +// 2-state interleaved path (`digest_pair`). // - s390x: `S390xPermuter` → portable permutation + KIMD batch-absorb. pub(crate) trait Permuter: Copy { @@ -669,13 +661,9 @@ impl Permuter for X86Permuter { /// keeps the portable scalar path for single-state and SHA3 CE for 2-state /// interleaved (`permute_x2`) when available. /// -/// The single-state SHA3 CE kernel is ~1.8× slower than portable on Neoverse -/// V1/V2 (Graviton3/4) — duplicating each u64 into both lanes of uint64x2_t -/// wastes half the SIMD bandwidth and incurs GPR↔NEON crossing penalties. On -/// Apple Silicon the SHA3 CE path is faster than the scalar named-register -/// permutation. The 2-state kernel packs two independent states lane-wise for -/// ~2× aggregate throughput, making SHA3 CE worthwhile on all aarch64 SHA3 CE -/// targets for parallel operations. +/// The single-state SHA3 CE kernel duplicates each `u64` across both vector +/// lanes. The 2-state kernel instead maps one independent state to each lane, +/// so parallel operations use the full vector width. #[cfg(all(target_arch = "aarch64", target_feature = "sha3", not(miri)))] #[derive(Clone, Copy, Default)] pub(crate) struct Aarch64Permuter; @@ -824,8 +812,7 @@ impl Permuter for Aarch64Permuter { #[inline(always)] fn permute_x2(self, state_a: &mut [u64; 25], state_b: &mut [u64; 25], len_hint: usize) { if self.has_sha3 { - // The 2-state kernel uses both NEON lanes meaningfully (state_a in - // lane 0, state_b in lane 1), achieving ~2× aggregate throughput. + // The 2-state kernel maps state_a to lane 0 and state_b to lane 1. aarch64::keccakf_aarch64_sha3_x2(state_a, state_b); } else { self.permute(state_a, len_hint); @@ -1265,8 +1252,7 @@ fn extract_output(state: &[u64; 25], out: &mut [u8; OUT]) { /// Hash two independent messages in parallel using 2-state interleaved /// permutation (aarch64 SHA3 CE) or sequential fallback. /// -/// On aarch64 with SHA3 CE, this achieves ~2× the aggregate throughput of -/// two sequential hash computations. +/// On AArch64 with SHA3 CE, one independent state occupies each vector lane. pub(crate) fn oneshot_pair( ds: u8, data_a: &[u8], @@ -1362,30 +1348,32 @@ fn xof_seeded_32_1_state(ds: u8, seed: &[u8; 32], x: u8) -> [ } #[cfg(feature = "ml-kem")] -pub(crate) fn xof_seeded_32_1(ds: u8, seed: &[u8; 32], x: u8) -> PublicKeccakXof { +pub(crate) fn xof_seeded_32_1_secret(ds: u8, seed: &[u8; 32], x: u8) -> KeccakXof { let permuter = PlatformPermuter::default(); let mut state = xof_seeded_32_1_state::(ds, seed, x); permuter.permute(&mut state, 0); - KeccakXofImpl { + let reader = KeccakXofImpl { state, pos: 0, permuter, - } + }; + crate::traits::ct::zeroize_words(&mut state); + reader } #[cfg(feature = "ml-kem")] -pub(crate) fn xof_seeded_32_1_pair( +pub(crate) fn xof_seeded_32_1_pair_secret( ds: u8, seed: &[u8; 32], a: u8, b: u8, -) -> (PublicKeccakXof, PublicKeccakXof) { +) -> (KeccakXof, KeccakXof) { let permuter = PlatformPermuter::default(); let mut state_a = xof_seeded_32_1_state::(ds, seed, a); let mut state_b = xof_seeded_32_1_state::(ds, seed, b); permuter.permute_x2(&mut state_a, &mut state_b, 0); - ( + let readers = ( KeccakXofImpl { state: state_a, pos: 0, @@ -1396,23 +1384,22 @@ pub(crate) fn xof_seeded_32_1_pair( pos: 0, permuter, }, - ) + ); + crate::traits::ct::zeroize_words_no_fence(&mut state_a); + crate::traits::ct::zeroize_words_no_fence(&mut state_b); + core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); + readers } #[cfg(feature = "ml-kem")] -pub(crate) fn xof_seeded_32_1_quad( +pub(crate) fn xof_seeded_32_1_quad_secret( ds: u8, seed: &[u8; 32], a: u8, b: u8, c: u8, d: u8, -) -> ( - PublicKeccakXof, - PublicKeccakXof, - PublicKeccakXof, - PublicKeccakXof, -) { +) -> (KeccakXof, KeccakXof, KeccakXof, KeccakXof) { let permuter = PlatformPermuter::default(); let mut state_a = xof_seeded_32_1_state::(ds, seed, a); let mut state_b = xof_seeded_32_1_state::(ds, seed, b); @@ -1420,7 +1407,7 @@ pub(crate) fn xof_seeded_32_1_quad( let mut state_d = xof_seeded_32_1_state::(ds, seed, d); permuter.permute_x4(&mut state_a, &mut state_b, &mut state_c, &mut state_d, 0); - ( + let readers = ( KeccakXofImpl { state: state_a, pos: 0, @@ -1441,7 +1428,13 @@ pub(crate) fn xof_seeded_32_1_quad( pos: 0, permuter, }, - ) + ); + crate::traits::ct::zeroize_words_no_fence(&mut state_a); + crate::traits::ct::zeroize_words_no_fence(&mut state_b); + crate::traits::ct::zeroize_words_no_fence(&mut state_c); + crate::traits::ct::zeroize_words_no_fence(&mut state_d); + core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); + readers } #[cfg(feature = "ml-kem")] diff --git a/src/hashes/crypto/keccak/aarch64.rs b/src/hashes/crypto/keccak/aarch64.rs index fba8e3ee..17cb826e 100644 --- a/src/hashes/crypto/keccak/aarch64.rs +++ b/src/hashes/crypto/keccak/aarch64.rs @@ -11,7 +11,7 @@ //! step — each saving 1–2 instructions vs the scalar equivalent. The ρ+π step uses scalar //! `rotate_left` which compiles to a single `ROR` instruction. //! - **2-state interleaved**: lane 0 = state A, lane 1 = state B. Processes two independent Keccak -//! states in parallel for ~2× aggregate throughput using full-width NEON SHA3 CE. +//! states together using full-width NEON SHA3 CE. //! //! # Safety //! diff --git a/src/hashes/crypto/keccak/dispatch_tables.rs b/src/hashes/crypto/keccak/dispatch_tables.rs index 9f2746f2..4047b11e 100644 --- a/src/hashes/crypto/keccak/dispatch_tables.rs +++ b/src/hashes/crypto/keccak/dispatch_tables.rs @@ -27,10 +27,9 @@ pub static DEFAULT_TABLE: DispatchTable = DispatchTable { l: KernelId::Portable, }; -// The SHA3 CE kernel for single-state is slower than portable on both -// Apple Silicon and Neoverse V1/V2 due to FMOV domain-crossing overhead -// between GPR↔NEON per SHA3 CE instruction. SHA3 CE only wins for the -// 2-state interleaved path (permute_x2) where both NEON lanes carry work. +// The single-state policy stays portable because it duplicates one state +// across both NEON lanes. SHA3 CE is used by the two-state interleaved path, +// where each lane carries an independent state. #[cfg(target_arch = "aarch64")] pub static AARCH64_SHA3_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, diff --git a/src/hashes/crypto/sha256/aarch64.rs b/src/hashes/crypto/sha256/aarch64.rs index 22ae2485..82fd0c87 100644 --- a/src/hashes/crypto/sha256/aarch64.rs +++ b/src/hashes/crypto/sha256/aarch64.rs @@ -248,8 +248,7 @@ pub(crate) unsafe fn compress_blocks_aarch64_sha2(state: &mut [u32; 8], blocks: // Rounds 16-63: compact loop (3 iterations × 16 rounds each). // - // Keep the message schedule interleaved with the hash rounds. That - // matches the faster sha2 crate kernel on Graviton3/4 and avoids + // Keep the message schedule interleaved with the hash rounds rather than // lengthening the schedule dependency chain ahead of the hash work. for t in (16..64).step_by(16) { s0 = vsha256su1q_u32(vsha256su0q_u32(s0, s1), s2, s3); diff --git a/src/hashes/crypto/sha256/mod.rs b/src/hashes/crypto/sha256/mod.rs index c33c1aa5..c67d238d 100644 --- a/src/hashes/crypto/sha256/mod.rs +++ b/src/hashes/crypto/sha256/mod.rs @@ -79,11 +79,10 @@ fn small_sigma1(x: u32) -> u32 { /// Read SHA-256 round constant K[i]. /// /// On x86/x86_64, 32-bit constants can be encoded as immediate operands in -/// `add r32, imm32`, so inlining is optimal. On all other architectures +/// `add r32, imm32`. On all other architectures /// (POWER, aarch64 portable, s390x, RISC-V), materializing a 32-bit /// immediate requires 2+ instructions (`lis`+`ori` on POWER, `movz`+`movk` -/// on ARM), so loading from the static K array via a single load instruction -/// is faster. +/// on ARM), so this path retains a load from the static K array. /// /// We use `core::hint::black_box` on the table base pointer instead of /// `core::ptr::read_volatile`. Both force the compiler to emit a memory @@ -96,8 +95,7 @@ fn small_sigma1(x: u32) -> u32 { /// `black_box`-based path generates the same single-load codegen on /// every target without `volatile`'s sharp edges. /// -/// This matches the `sha2` crate's `rk()` strategy and closes the 15-20% -/// gap on POWER10. +/// This matches the `sha2` crate's `rk()` strategy. #[inline(always)] fn rk(i: usize) -> u32 { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] @@ -133,8 +131,7 @@ pub(crate) fn compress_block_with( small_s0: fn(u32) -> u32, small_s1: fn(u32) -> u32, ) { - // 16-word ring buffer message schedule (lower memory traffic than a full - // 64-word schedule, and typically faster in practice). + // 16-word ring-buffer schedule instead of a full 64-word schedule. // // Fully unrolled to avoid bounds checks and allow better instruction // scheduling in the scalar core. diff --git a/src/hashes/crypto/sha3.rs b/src/hashes/crypto/sha3.rs index ec60a9fb..b71a3b4e 100644 --- a/src/hashes/crypto/sha3.rs +++ b/src/hashes/crypto/sha3.rs @@ -4,16 +4,165 @@ #[cfg(all(test, feature = "ml-kem"))] use super::keccak::xof_quad; -use super::keccak::{PublicKeccakCore, PublicKeccakXof}; #[cfg(feature = "ml-kem")] use super::keccak::{ - xof_seeded_32_1 as keccak_xof_seeded_32_1, xof_seeded_32_1_pair as keccak_xof_seeded_32_1_pair, - xof_seeded_32_1_quad as keccak_xof_seeded_32_1_quad, xof_seeded_32_2 as keccak_xof_seeded_32_2, + KeccakCore, KeccakXof, xof_seeded_32_1_pair_secret as keccak_xof_seeded_32_1_pair_secret, + xof_seeded_32_1_quad_secret as keccak_xof_seeded_32_1_quad_secret, + xof_seeded_32_1_secret as keccak_xof_seeded_32_1_secret, xof_seeded_32_2 as keccak_xof_seeded_32_2, xof_seeded_32_2_pair as keccak_xof_seeded_32_2_pair, xof_seeded_32_2_quad as keccak_xof_seeded_32_2_quad, xof_seeded_32_2_triple as keccak_xof_seeded_32_2_triple, }; +use super::keccak::{PublicKeccakCore, PublicKeccakXof}; use crate::traits::{Digest, Xof}; +#[cfg(feature = "ml-kem")] +pub(crate) fn mlkem_sha3_512_digest(input: &[u8]) -> [u8; 64] { + let mut core = KeccakCore::<72>::default(); + core.update(input); + let mut out = [0u8; 64]; + core.finalize_into_fixed(0x06, &mut out); + out +} + +#[cfg(feature = "ml-kem")] +pub(crate) fn mlkem_shake256_two_part_into(a: &[u8], b: &[u8], out: &mut [u8]) { + let mut core = KeccakCore::<136>::default(); + core.update(a); + core.update(b); + let mut reader = core.finalize_xof(0x1F); + reader.squeeze_into(out); +} + +#[cfg(feature = "ml-kem")] +pub(crate) struct MlKemShake256XofReader { + inner: KeccakXof<136>, +} + +#[cfg(feature = "ml-kem")] +impl MlKemShake256XofReader { + #[inline] + pub(crate) fn seeded_32_1(seed: &[u8; 32], nonce: u8) -> Self { + Self { + inner: keccak_xof_seeded_32_1_secret::<136>(0x1F, seed, nonce), + } + } + + #[inline] + pub(crate) fn seeded_32_1_pair(seed: &[u8; 32], a: u8, b: u8) -> (Self, Self) { + let (a, b) = keccak_xof_seeded_32_1_pair_secret::<136>(0x1F, seed, a, b); + (Self { inner: a }, Self { inner: b }) + } + + #[inline] + pub(crate) fn seeded_32_1_quad(seed: &[u8; 32], a: u8, b: u8, c: u8, d: u8) -> (Self, Self, Self, Self) { + let (a, b, c, d) = keccak_xof_seeded_32_1_quad_secret::<136>(0x1F, seed, a, b, c, d); + ( + Self { inner: a }, + Self { inner: b }, + Self { inner: c }, + Self { inner: d }, + ) + } + + #[inline] + pub(crate) fn squeeze(&mut self, out: &mut [u8]) { + self.inner.squeeze_into(out); + } + + #[inline] + pub(crate) fn squeeze_pair(a: &mut Self, b: &mut Self, out_a: &mut [u8], out_b: &mut [u8]) { + KeccakXof::<136>::squeeze_pair_into(&mut a.inner, &mut b.inner, out_a, out_b); + } + + #[inline] + #[allow(clippy::too_many_arguments)] + pub(crate) fn squeeze_quad( + a: &mut Self, + b: &mut Self, + c: &mut Self, + d: &mut Self, + out_a: &mut [u8], + out_b: &mut [u8], + out_c: &mut [u8], + out_d: &mut [u8], + ) { + KeccakXof::<136>::squeeze_quad_into( + &mut a.inner, + &mut b.inner, + &mut c.inner, + &mut d.inner, + out_a, + out_b, + out_c, + out_d, + ); + } +} + +#[cfg(all(feature = "diag", feature = "ml-kem"))] +#[doc(hidden)] +#[unsafe(no_mangle)] +#[inline(never)] +pub fn diag_zeroize_mlkem_sha3_512(mut seed: [u8; 32]) -> u8 { + let mut digest = mlkem_sha3_512_digest(&seed); + crate::traits::ct::zeroize(&mut seed); + let observed = digest[0]; + crate::traits::ct::zeroize(&mut digest); + core::hint::black_box(observed) +} + +#[cfg(all(feature = "diag", feature = "ml-kem"))] +#[doc(hidden)] +#[unsafe(no_mangle)] +#[inline(never)] +pub fn diag_zeroize_mlkem_shake256_scalar(mut seed: [u8; 32]) -> u8 { + let mut reader = MlKemShake256XofReader::seeded_32_1(&seed, 1); + crate::traits::ct::zeroize(&mut seed); + let mut out = [0u8; 192]; + reader.squeeze(&mut out); + let observed = out[0]; + crate::traits::ct::zeroize(&mut out); + core::hint::black_box(observed) +} + +#[cfg(all(feature = "diag", feature = "ml-kem"))] +#[doc(hidden)] +#[unsafe(no_mangle)] +#[inline(never)] +pub fn diag_zeroize_mlkem_shake256_pair(mut seed: [u8; 32]) -> u8 { + let (mut a, mut b) = MlKemShake256XofReader::seeded_32_1_pair(&seed, 1, 2); + crate::traits::ct::zeroize(&mut seed); + let mut out_a = [0u8; 192]; + let mut out_b = [0u8; 192]; + MlKemShake256XofReader::squeeze_pair(&mut a, &mut b, &mut out_a, &mut out_b); + let observed = out_a[0] ^ out_b[0]; + crate::traits::ct::zeroize(&mut out_a); + crate::traits::ct::zeroize(&mut out_b); + core::hint::black_box(observed) +} + +#[cfg(all(feature = "diag", feature = "ml-kem"))] +#[doc(hidden)] +#[unsafe(no_mangle)] +#[inline(never)] +pub fn diag_zeroize_mlkem_shake256_quad(mut seed: [u8; 32]) -> u8 { + let (mut a, mut b, mut c, mut d) = MlKemShake256XofReader::seeded_32_1_quad(&seed, 1, 2, 3, 4); + crate::traits::ct::zeroize(&mut seed); + let mut out_a = [0u8; 192]; + let mut out_b = [0u8; 192]; + let mut out_c = [0u8; 192]; + let mut out_d = [0u8; 192]; + MlKemShake256XofReader::squeeze_quad( + &mut a, &mut b, &mut c, &mut d, &mut out_a, &mut out_b, &mut out_c, &mut out_d, + ); + let observed = out_a[0] ^ out_b[0] ^ out_c[0] ^ out_d[0]; + crate::traits::ct::zeroize(&mut out_a); + crate::traits::ct::zeroize(&mut out_b); + crate::traits::ct::zeroize(&mut out_c); + crate::traits::ct::zeroize(&mut out_d); + core::hint::black_box(observed) +} + /// SHA3-256 digest state. /// /// Standardized in FIPS 202. @@ -99,9 +248,8 @@ impl Digest for Sha3_224 { impl Sha3_224 { /// Hash two independent messages in parallel, returning both 28-byte digests. /// - /// On aarch64 with SHA3 Crypto Extensions, this achieves ~2× the aggregate - /// throughput of two sequential [`digest`](Digest::digest) calls by using - /// 2-state NEON interleaving. + /// The aarch64 SHA3 Crypto Extensions backend interleaves the two independent + /// states. #[inline] #[must_use] pub fn digest_pair(a: &[u8], b: &[u8]) -> ([u8; 28], [u8; 28]) { @@ -144,9 +292,8 @@ impl Digest for Sha3_256 { impl Sha3_256 { /// Hash two independent messages in parallel, returning both 32-byte digests. /// - /// On aarch64 with SHA3 Crypto Extensions, this achieves ~2× the aggregate - /// throughput of two sequential [`digest`](Digest::digest) calls by using - /// 2-state NEON interleaving. + /// The aarch64 SHA3 Crypto Extensions backend interleaves the two independent + /// states. #[inline] #[must_use] pub fn digest_pair(a: &[u8], b: &[u8]) -> ([u8; 32], [u8; 32]) { @@ -239,9 +386,8 @@ impl Digest for Sha3_384 { impl Sha3_384 { /// Hash two independent messages in parallel, returning both 48-byte digests. /// - /// On aarch64 with SHA3 Crypto Extensions, this achieves ~2× the aggregate - /// throughput of two sequential [`digest`](Digest::digest) calls by using - /// 2-state NEON interleaving. + /// The aarch64 SHA3 Crypto Extensions backend interleaves the two independent + /// states. #[inline] #[must_use] pub fn digest_pair(a: &[u8], b: &[u8]) -> ([u8; 48], [u8; 48]) { @@ -284,9 +430,8 @@ impl Digest for Sha3_512 { impl Sha3_512 { /// Hash two independent messages in parallel, returning both 64-byte digests. /// - /// On aarch64 with SHA3 Crypto Extensions, this achieves ~2× the aggregate - /// throughput of two sequential [`digest`](Digest::digest) calls by using - /// 2-state NEON interleaving. + /// The aarch64 SHA3 Crypto Extensions backend interleaves the two independent + /// states. #[inline] #[must_use] pub fn digest_pair(a: &[u8], b: &[u8]) -> ([u8; 64], [u8; 64]) { @@ -627,44 +772,6 @@ impl Shake256 { *self = Self::default(); } - #[inline] - #[cfg(feature = "ml-kem")] - pub(crate) fn xof_seeded_32_1(seed: &[u8; 32], x: u8) -> Shake256XofReader { - Shake256XofReader { - inner: keccak_xof_seeded_32_1::<136>(0x1F, seed, x), - } - } - - #[inline] - #[cfg(feature = "ml-kem")] - pub(crate) fn xof_seeded_32_1_pair(seed: &[u8; 32], a: u8, b: u8) -> (Shake256XofReader, Shake256XofReader) { - let (a, b) = keccak_xof_seeded_32_1_pair::<136>(0x1F, seed, a, b); - (Shake256XofReader { inner: a }, Shake256XofReader { inner: b }) - } - - #[inline] - #[cfg(feature = "ml-kem")] - pub(crate) fn xof_seeded_32_1_quad( - seed: &[u8; 32], - a: u8, - b: u8, - c: u8, - d: u8, - ) -> ( - Shake256XofReader, - Shake256XofReader, - Shake256XofReader, - Shake256XofReader, - ) { - let (a, b, c, d) = keccak_xof_seeded_32_1_quad::<136>(0x1F, seed, a, b, c, d); - ( - Shake256XofReader { inner: a }, - Shake256XofReader { inner: b }, - Shake256XofReader { inner: c }, - Shake256XofReader { inner: d }, - ) - } - #[inline] /// Convenience one-shot XOF output for callers that only need bytes. /// @@ -693,44 +800,13 @@ impl Xof for Shake256XofReader { } } -impl Shake256XofReader { - #[inline] - #[cfg(feature = "ml-kem")] - pub(crate) fn squeeze_pair(a: &mut Self, b: &mut Self, out_a: &mut [u8], out_b: &mut [u8]) { - PublicKeccakXof::<136>::squeeze_pair_into(&mut a.inner, &mut b.inner, out_a, out_b); - } - - #[inline] - #[cfg(feature = "ml-kem")] - #[allow(clippy::too_many_arguments)] - pub(crate) fn squeeze_quad( - a: &mut Self, - b: &mut Self, - c: &mut Self, - d: &mut Self, - out_a: &mut [u8], - out_b: &mut [u8], - out_c: &mut [u8], - out_d: &mut [u8], - ) { - PublicKeccakXof::<136>::squeeze_quad_into( - &mut a.inner, - &mut b.inner, - &mut c.inner, - &mut d.inner, - out_a, - out_b, - out_c, - out_d, - ); - } -} - impl_xof_read!(Shake256XofReader); #[cfg(test)] mod tests { - use super::{Sha3_224, Sha3_256, Sha3_384, Sha3_512, Shake128, Shake128XofReader, Shake256}; + #[cfg(feature = "ml-kem")] + use super::Shake128XofReader; + use super::{Sha3_224, Sha3_256, Sha3_384, Sha3_512, Shake128, Shake256}; use crate::traits::{Digest, Xof}; fn hex(bytes: &[u8]) -> alloc::string::String { diff --git a/src/hashes/crypto/sha384.rs b/src/hashes/crypto/sha384.rs index a48b6a55..53720dea 100644 --- a/src/hashes/crypto/sha384.rs +++ b/src/hashes/crypto/sha384.rs @@ -38,6 +38,12 @@ pub(crate) const H0: [u64; 8] = [ /// /// Standardized in FIPS 180-4. /// +/// # Panics +/// +/// [`Digest::update`] and its vectored variants panic before absorbing an +/// update that would make the cumulative input exceed the FIPS 180-4 maximum +/// of 2^125 − 1 bytes. +/// /// # Examples /// /// ``` @@ -125,10 +131,7 @@ impl Sha384 { } }; - let total = self - .bytes_hashed - .strict_add(self.block_len as u128) - .strict_add(incoming_len as u128); + let total = Sha512::checked_total_len(self.bytes_hashed, self.block_len, incoming_len); let compress = dispatch.select(len_hint_from_u128(total)); self.compress_blocks = compress; compress diff --git a/src/hashes/crypto/sha512/dispatch_tables.rs b/src/hashes/crypto/sha512/dispatch_tables.rs index 0a7a9d29..f5bd5fb6 100644 --- a/src/hashes/crypto/sha512/dispatch_tables.rs +++ b/src/hashes/crypto/sha512/dispatch_tables.rs @@ -100,18 +100,11 @@ pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { #[inline] #[must_use] pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static DispatchTable { - // x86_64 cascade: SHA-512 NI > vendor-aware AVX2/AVX-512VL > Portable + // x86_64 cascade: SHA-512 NI, then vendor-aware AVX2/AVX-512VL, then portable. // - // The stitched AVX2+BMI2 dual-block kernel beats AVX-512VL in raw - // compression throughput on both AMD and Intel. However, the AVX2 kernel - // falls back to portable for odd-block-count inputs (the common case for - // small inputs: 0-64 B = 1 block). On Intel, AVX-512VL handles single - // blocks natively, so it wins at small sizes and breaks even at scale. - // - // AMD: AVX2 decoupled > AVX-512VL. - // Intel: AVX-512VL decoupled > AVX2 decoupled. - // - // Measured: sha512-compress/raw CI 2026-03-23. + // The AVX2 kernel falls back to portable for odd block counts, while + // AVX-512VL handles a trailing single block. Vendor-specific ordering is a + // manually maintained dispatch policy. #[cfg(target_arch = "x86_64")] { use crate::platform::caps::x86; @@ -123,7 +116,6 @@ pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static D // serialises schedule → extract → round within each iteration, limiting // IPC on wide pipelines. The decoupled pattern gives the OOO engine // 16 independent scalar rounds to overlap with SIMD schedule latency. - // Measured: stitched Zen4→Zen5 scaling 1.32x vs sha2 crate 1.71x. if caps.has(x86::AVX2) { return &X86_AVX2_DECOUPLED_TABLE; } diff --git a/src/hashes/crypto/sha512/mod.rs b/src/hashes/crypto/sha512/mod.rs index ec9c0e4e..65459e27 100644 --- a/src/hashes/crypto/sha512/mod.rs +++ b/src/hashes/crypto/sha512/mod.rs @@ -36,6 +36,9 @@ pub(crate) mod x86_64_avx512vl; const BLOCK_LEN: usize = 128; +/// Maximum SHA-512-family message length in bytes under the 128-bit bit-length field. +pub(crate) const MAX_MESSAGE_LEN: u128 = u128::MAX / 8; + pub(crate) const H0: [u64; 8] = [ 0x6a09_e667_f3bc_c908, 0xbb67_ae85_84ca_a73b, @@ -169,6 +172,12 @@ fn small_sigma1(x: u64) -> u64 { /// /// Standardized in FIPS 180-4. /// +/// # Panics +/// +/// [`Digest::update`] and its vectored variants panic before absorbing an +/// update that would make the cumulative input exceed the FIPS 180-4 maximum +/// of 2^125 − 1 bytes. +/// /// # Examples /// /// ``` @@ -233,6 +242,18 @@ impl Default for Sha512 { } impl Sha512 { + #[inline] + pub(crate) fn checked_total_len(bytes_hashed: u128, block_len: usize, incoming_len: usize) -> u128 { + let total = bytes_hashed + .strict_add(block_len as u128) + .strict_add(incoming_len as u128); + assert!( + total <= MAX_MESSAGE_LEN, + "SHA-512 family: total input exceeds FIPS 180-4 maximum of 2^125 − 1 bytes" + ); + total + } + /// Compute the digest of `data` in one shot. /// /// This selects the best available kernel for the current platform and input @@ -280,10 +301,7 @@ impl Sha512 { } }; - let total = self - .bytes_hashed - .strict_add(self.block_len as u128) - .strict_add(incoming_len as u128); + let total = Self::checked_total_len(self.bytes_hashed, self.block_len, incoming_len); let compress = dispatch.select(len_hint_from_u128(total)); self.compress_blocks = compress; compress @@ -401,7 +419,7 @@ impl Sha512 { total_len: u128, compress_blocks: CompressBlocksFn, ) -> [u64; 8] { - let bit_len = total_len << 3; + let bit_len = total_len.strict_mul(8); block[block_len] = 0x80; block_len = block_len.strict_add(1); @@ -846,7 +864,7 @@ impl_std_io_write_for_digest!(Sha512); #[cfg(test)] mod tests { - use super::Sha512; + use super::{MAX_MESSAGE_LEN, Sha512}; fn hex64(bytes: &[u8; 64]) -> alloc::string::String { use alloc::string::String; @@ -870,4 +888,15 @@ mod tests { "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" ); } + + #[test] + fn message_length_boundary_is_checked_before_absorption() { + assert_eq!(Sha512::checked_total_len(MAX_MESSAGE_LEN, 0, 0), MAX_MESSAGE_LEN); + } + + #[test] + #[should_panic(expected = "total input exceeds FIPS 180-4 maximum")] + fn message_length_above_boundary_panics() { + let _ = Sha512::checked_total_len(MAX_MESSAGE_LEN, 0, 1); + } } diff --git a/src/hashes/crypto/sha512/x86_64_avx2.rs b/src/hashes/crypto/sha512/x86_64_avx2.rs index 87be2a8c..993da189 100644 --- a/src/hashes/crypto/sha512/x86_64_avx2.rs +++ b/src/hashes/crypto/sha512/x86_64_avx2.rs @@ -137,8 +137,7 @@ unsafe fn extract_128(v: __m128i) -> (u64, u64) { /// /// Same approach as [`schedule_rotate_256`] but for 128-bit registers. /// Eliminates ring-buffer index computation (`wrapping_sub`, `& 7`) in favour -/// of fixed-offset array accesses + physical rotation. The 7 register moves -/// are zero-cost on modern x86 (register renaming). +/// of fixed-offset array accesses and physical rotation. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn schedule_rotate_128(x: &mut [__m128i; 8], k: __m128i) -> __m128i { @@ -173,9 +172,7 @@ unsafe fn schedule_rotate_128(x: &mut [__m128i; 8], k: __m128i) -> __m128i { /// /// Physically rotates the `x[]` array so that adjacent schedule words are /// always in adjacent registers. This lets `_mm256_alignr_epi8` extract -/// cross-register values without `_mm256_permute2x128_si256` (3-cycle latency -/// on Zen 5). The 7 register moves from rotation are zero-cost on modern x86 -/// (register renaming). +/// cross-register values without `_mm256_permute2x128_si256`. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn schedule_rotate_256(x: &mut [__m256i; 8], k: __m256i) -> __m256i { @@ -193,7 +190,7 @@ unsafe fn schedule_rotate_256(x: &mut [__m256i; 8], k: __m256i) -> __m256i { ); // Rotate: x[0] (newest) → x[7], shift everything left. - // Zero-cost on modern x86 via register renaming. + // Rotate the logical ring to keep adjacent words in adjacent registers. let new_val = x[0]; x[0] = x[1]; x[1] = x[2]; @@ -221,8 +218,7 @@ unsafe fn schedule_rotate_256(x: &mut [__m256i; 8], k: __m256i) -> __m256i { /// /// 2. **Rotation-based schedule** — the `w[]` array is physically rotated after each schedule /// update so adjacent words stay in adjacent registers. This lets `_mm256_alignr_epi8` extract -/// cross-register values directly, eliminating `_mm256_permute2x128_si256` (3-cycle latency on -/// Zen 5). The 7 register moves from rotation are zero-cost (register renaming). +/// cross-register values directly, eliminating `_mm256_permute2x128_si256`. /// /// Uses the **standard round** (Σ0 and Σ1 computed independently within /// each round for maximum within-round parallelism). diff --git a/src/hashes/crypto/sha512/x86_64_avx512vl.rs b/src/hashes/crypto/sha512/x86_64_avx512vl.rs index 3e17172c..9d29c714 100644 --- a/src/hashes/crypto/sha512/x86_64_avx512vl.rs +++ b/src/hashes/crypto/sha512/x86_64_avx512vl.rs @@ -15,8 +15,8 @@ //! - **Single-block** (odd trailing): 128-bit schedule (`__m128i` ring buffer, VPRORQ) stitched //! with scalar rounds. No portable fallback — fully self-contained. //! -//! VPRORQ (`_mm256_ror_epi64` / `_mm_ror_epi64`) provides 1-instruction 64-bit rotates vs -//! 3-instruction shift-shift-or on AVX2, yielding ~40% fewer SIMD schedule instructions. +//! VPRORQ (`_mm256_ror_epi64` / `_mm_ror_epi64`) provides native 64-bit +//! rotates instead of shift/shift/or emulation. //! 256-bit register width avoids frequency throttling on Intel. //! //! BMI2 (`bmi2` target feature) enables `RORX` for scalar rotations. @@ -452,8 +452,8 @@ unsafe fn schedule_rotate_128(x: &mut [__m128i; 8], k: __m128i) -> __m128i { /// /// Uses VPRORQ native rotates for sigma functions, eliminating the 3-op /// shift-shift-or of the AVX2 path. Combined with array rotation, this also -/// eliminates `_mm256_permute2x128_si256` (3-cycle latency on SPR) that the -/// ring-buffer schedule uses for cross-lane extraction. +/// eliminates `_mm256_permute2x128_si256`, which the ring-buffer schedule uses +/// for cross-lane extraction. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn schedule_rotate_256(x: &mut [__m256i; 8], k: __m256i) -> __m256i { @@ -471,7 +471,6 @@ unsafe fn schedule_rotate_256(x: &mut [__m256i; 8], k: __m256i) -> __m256i { ); // Rotate: x[0] (newest) → x[7], shift everything left. - // Zero-cost on modern x86 via register renaming. let new_val = x[0]; x[0] = x[1]; x[1] = x[2]; @@ -499,11 +498,10 @@ unsafe fn schedule_rotate_256(x: &mut [__m256i; 8], k: __m256i) -> __m256i { /// /// 2. **Rotation-based schedule** — the `w[]` array is physically rotated after each schedule /// update so adjacent words stay in adjacent registers. This lets `_mm256_alignr_epi8` extract -/// cross-register values directly, eliminating `_mm256_permute2x128_si256` (3-cycle latency on -/// SPR). The 7 register moves from rotation are zero-cost (register renaming). +/// cross-register values directly, eliminating `_mm256_permute2x128_si256`. /// -/// 3. **VPRORQ native rotates** — single-instruction 64-bit vector rotates for sigma functions (1 -/// instruction vs 3-op shift-shift-or on AVX2), yielding ~40% fewer SIMD schedule instructions. +/// 3. **VPRORQ native rotates** — native 64-bit vector rotates for sigma functions instead of +/// shift/shift/or emulation. /// /// Uses the **standard round** (Σ0 and Σ1 computed independently within /// each round for maximum within-round parallelism). diff --git a/src/hashes/crypto/sha512_256.rs b/src/hashes/crypto/sha512_256.rs index e908a114..baa1d8e9 100644 --- a/src/hashes/crypto/sha512_256.rs +++ b/src/hashes/crypto/sha512_256.rs @@ -38,6 +38,12 @@ const H0: [u64; 8] = [ /// /// Standardized in FIPS 180-4. /// +/// # Panics +/// +/// [`Digest::update`] and its vectored variants panic before absorbing an +/// update that would make the cumulative input exceed the FIPS 180-4 maximum +/// of 2^125 − 1 bytes. +/// /// # Examples /// /// ``` @@ -102,10 +108,7 @@ impl Sha512_256 { } }; - let total = self - .bytes_hashed - .strict_add(self.block_len as u128) - .strict_add(incoming_len as u128); + let total = Sha512::checked_total_len(self.bytes_hashed, self.block_len, incoming_len); let compress = dispatch.select(len_hint_from_u128(total)); self.compress_blocks = compress; compress diff --git a/src/hashes/crypto/sp800185.rs b/src/hashes/crypto/sp800185.rs index c1b68f94..0e45cbaf 100644 --- a/src/hashes/crypto/sp800185.rs +++ b/src/hashes/crypto/sp800185.rs @@ -56,6 +56,7 @@ pub(crate) fn absorb_bytepad(core: &mut KeccakCore, seg let total_len = prefix_len.strict_add(payload_len); let rem = total_len % RATE; - let pad_len = if rem == 0 { RATE } else { RATE.strict_sub(rem) }; - core.update(&[0u8; RATE][..pad_len]); + if rem != 0 { + core.update(&[0u8; RATE][..RATE.strict_sub(rem)]); + } } diff --git a/src/hashes/fast/xxh3.rs b/src/hashes/fast/xxh3.rs index fa213c1e..402990d3 100644 --- a/src/hashes/fast/xxh3.rs +++ b/src/hashes/fast/xxh3.rs @@ -1,8 +1,7 @@ //! XXH3 (**NOT CRYPTO**). //! //! Hardware-accelerated on x86-64 (AVX2, AVX-512), aarch64 (NEON), POWER -//! (VSX), s390x (z/Vector), and WASM (SIMD128), with a portable scalar -//! fallback. +//! (VSX), and s390x (z/Vector), with a portable scalar fallback. #![allow(clippy::indexing_slicing)] // Tight block parsing + fixed-size arrays @@ -114,8 +113,7 @@ unsafe fn read_u64_le(input: &[u8], offset: usize) -> u64 { /// Extract a pair of 8-byte arrays at `offset` from a byte slice. /// -/// Matches xxhash-rust's `get_aligned_chunk_ref::<[[u8; 8]; 2]>` pattern -/// for optimal codegen: `from_ne_bytes(chunk[0])` compiles to a single `ldr`. +/// Matches xxhash-rust's `get_aligned_chunk_ref::<[[u8; 8]; 2]>` call shape. /// /// # Safety /// @@ -579,25 +577,6 @@ pub(crate) fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { } } -#[cfg(any(test, feature = "diag"))] -#[inline(always)] -fn xxh3_64_with_seed(input: &[u8], seed: u64) -> u64 { - let len = input.len(); - if len <= 16 { - xxh3_64_0to16(input, seed, &DEFAULT_SECRET) - } else if len == 32 { - xxh3_64_32(input, seed, &DEFAULT_SECRET) - } else if len == 64 { - xxh3_64_64(input, seed, &DEFAULT_SECRET) - } else if len <= 128 { - xxh3_64_7to128(input, seed, &DEFAULT_SECRET) - } else if len <= MID_SIZE_MAX { - xxh3_64_129to240(input, seed, &DEFAULT_SECRET) - } else { - xxh3_64_long(input, seed) - } -} - #[inline(always)] fn xxh3_128_1to3(input: &[u8], seed: u64, secret: &[u8]) -> u128 { // SAFETY: callers ensure input.len() is 1..=3 and secret.len() >= SECRET_SIZE_MIN. diff --git a/src/hashes/fast/xxh3/aarch64_neon.rs b/src/hashes/fast/xxh3/aarch64_neon.rs index 00a83910..d9f0b4b9 100644 --- a/src/hashes/fast/xxh3/aarch64_neon.rs +++ b/src/hashes/fast/xxh3/aarch64_neon.rs @@ -218,7 +218,11 @@ unsafe fn stream_accumulate_inner( } } -pub(crate) fn stream_accumulate( +/// # Safety +/// +/// The caller must verify NEON support, pass `stripes > 0`, and keep every +/// requested input and secret stripe within its slice. +pub(crate) unsafe fn stream_accumulate( initial: [u64; ACC_NB], input: &[u8], input_offset: usize, @@ -227,9 +231,8 @@ pub(crate) fn stream_accumulate( stripes: usize, scramble_after: bool, ) -> [u64; ACC_NB] { - // SAFETY: Calling the NEON streaming kernel because: - // 1. Runtime dispatch selects this wrapper only on aarch64 with NEON capability. - // 2. `stream_accumulate_inner` validates all slice offsets before pointer arithmetic. + // SAFETY: The caller contract provides NEON support and proves every raw + // input and secret load is in bounds. unsafe { stream_accumulate_inner( initial, @@ -355,23 +358,6 @@ pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { } } -#[cfg(any(test, feature = "diag"))] -/// XXH3 64-bit hash — NEON kernel. -/// -/// Delegates ≤240 B to portable scalar paths; >240 B uses NEON accumulator. -pub fn xxh3_64_with_seed(input: &[u8], seed: u64) -> u64 { - if input.len() <= 16 { - return super::xxh3_64_0to16(input, seed, &DEFAULT_SECRET); - } - if input.len() <= 128 { - return super::xxh3_64_7to128(input, seed, &DEFAULT_SECRET); - } - if input.len() <= super::MID_SIZE_MAX { - return super::xxh3_64_129to240(input, seed, &DEFAULT_SECRET); - } - xxh3_64_long(input, seed) -} - /// Long-path entry point (>240B) — no ≤240B branches. pub fn xxh3_128_long_default(input: &[u8]) -> u128 { // SAFETY: NEON always available on aarch64. diff --git a/src/hashes/fast/xxh3/dispatch.rs b/src/hashes/fast/xxh3/dispatch.rs index de190497..0c1b4be4 100644 --- a/src/hashes/fast/xxh3/dispatch.rs +++ b/src/hashes/fast/xxh3/dispatch.rs @@ -1,7 +1,5 @@ #![cfg_attr(test, allow(dead_code))] -#[cfg(any(test, feature = "diag"))] -use super::kernels::hash64_fn; use super::{ dispatch_tables::DispatchTable, kernels::{ @@ -22,23 +20,7 @@ struct ActiveDispatch { long128: Hash128Fn, stream_accumulate: StreamAccumulateFn, #[cfg(any(test, feature = "diag"))] - boundaries: [usize; 3], - #[cfg(any(test, feature = "diag"))] - xs64: Hash64Fn, - #[cfg(any(test, feature = "diag"))] - s64: Hash64Fn, - #[cfg(any(test, feature = "diag"))] - m64: Hash64Fn, - #[cfg(any(test, feature = "diag"))] - l64: Hash64Fn, - #[cfg(any(test, feature = "diag"))] - xs_name: &'static str, - #[cfg(any(test, feature = "diag"))] - s_name: &'static str, - #[cfg(any(test, feature = "diag"))] - m_name: &'static str, - #[cfg(any(test, feature = "diag"))] - l_name: &'static str, + long_id: Xxh3KernelId, } static ACTIVE: OnceCache = OnceCache::new(); @@ -72,36 +54,14 @@ fn active() -> ActiveDispatch { let caps = crate::platform::caps(); let table: &'static DispatchTable = super::dispatch_tables::select_runtime_table(caps); - #[cfg(any(test, feature = "diag"))] - let xs_id = resolve(table.xs, caps); - #[cfg(any(test, feature = "diag"))] - let s_id = resolve(table.s, caps); - #[cfg(any(test, feature = "diag"))] - let m_id = resolve(table.m, caps); - let l_id = resolve(table.l, caps); + let long_id = resolve(table.long, caps); ActiveDispatch { - long64: hash64_long_fn(l_id), - long128: hash128_long_fn(l_id), - stream_accumulate: kernel_stream_accumulate_fn(l_id), - #[cfg(any(test, feature = "diag"))] - boundaries: table.boundaries, + long64: hash64_long_fn(long_id), + long128: hash128_long_fn(long_id), + stream_accumulate: kernel_stream_accumulate_fn(long_id), #[cfg(any(test, feature = "diag"))] - xs64: hash64_fn(xs_id), - #[cfg(any(test, feature = "diag"))] - s64: hash64_fn(s_id), - #[cfg(any(test, feature = "diag"))] - m64: hash64_fn(m_id), - #[cfg(any(test, feature = "diag"))] - l64: hash64_fn(l_id), - #[cfg(any(test, feature = "diag"))] - xs_name: xs_id.as_str(), - #[cfg(any(test, feature = "diag"))] - s_name: s_id.as_str(), - #[cfg(any(test, feature = "diag"))] - m_name: m_id.as_str(), - #[cfg(any(test, feature = "diag"))] - l_name: l_id.as_str(), + long_id, } }) } @@ -114,33 +74,46 @@ pub(crate) fn stream_accumulate_fn() -> StreamAccumulateFn { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] -fn select64(d: &ActiveDispatch, len: usize) -> (Hash64Fn, &'static str) { - let [xs_max, s_max, m_max] = d.boundaries; - if len <= xs_max { - (d.xs64, d.xs_name) - } else if len <= s_max { - (d.s64, d.s_name) - } else if len <= m_max { - (d.m64, d.m_name) - } else { - (d.l64, d.l_name) +fn kernel_id64_for_len(long_id: Xxh3KernelId, caps: Caps, len: usize) -> Xxh3KernelId { + if len <= super::MID_SIZE_MAX { + return Xxh3KernelId::Portable; } + + #[cfg(target_arch = "x86_64")] + if len <= ZEN5_XXH3_64_AVX2_LONG_MAX + && caps.has(crate::platform::caps::x86::AMD_ZEN5 | crate::platform::caps::x86::AVX2) + { + return Xxh3KernelId::Avx2; + } + + let _ = caps; + long_id } #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] -pub fn kernel_name_for_len(len: usize) -> &'static str { +pub fn kernel_name64_for_len(len: usize) -> &'static str { let d = active(); - select64(&d, len).1 + kernel_id64_for_len(d.long_id, crate::platform::caps(), len).as_str() +} + +#[cfg(any(test, feature = "diag"))] +#[inline] +#[must_use] +pub fn kernel_name128_for_len(len: usize) -> &'static str { + if len <= super::MID_SIZE_MAX { + Xxh3KernelId::Portable.as_str() + } else { + active().long_id.as_str() + } } /// Flat size-based dispatch matching xxhash-rust's branch structure. /// /// Every SIMD kernel delegates inputs ≤ 240 B back to the same portable scalar /// functions, so we dispatch directly to the sub-functions here — eliminating -/// the intermediate `xxh3_64_with_seed` call and its redundant ≤`MID_SIZE_MAX` -/// guard branch. +/// redundant ≤`MID_SIZE_MAX` checks in the long-path kernels. /// /// Compile-time long dispatch inlines to a direct kernel call. Runtime fallback /// stays out-of-line so ≤240B paths do not carry the cache/feature lookup. @@ -407,3 +380,32 @@ fn hash128_long_runtime(seed: u64, data: &[u8]) -> u128 { let d = active(); (d.long128)(data, seed) } + +#[cfg(all(test, target_arch = "x86_64"))] +mod tests { + use super::*; + use crate::platform::caps::x86; + + #[test] + fn diagnostic_kernel_ids_match_short_and_zen5_production_policy() { + for len in [0, 16, 64, 128, 240] { + assert_eq!( + kernel_id64_for_len(Xxh3KernelId::Avx512, x86::AVX512F, len), + Xxh3KernelId::Portable + ); + } + + let zen5 = x86::AMD_ZEN5 | x86::AVX2 | x86::AVX512F; + for len in [241, 256, 1024] { + assert_eq!(kernel_id64_for_len(Xxh3KernelId::Avx512, zen5, len), Xxh3KernelId::Avx2); + } + assert_eq!( + kernel_id64_for_len(Xxh3KernelId::Avx512, zen5, 1025), + Xxh3KernelId::Avx512 + ); + assert_eq!( + kernel_id64_for_len(Xxh3KernelId::Avx512, x86::AVX512F, 241), + Xxh3KernelId::Avx512 + ); + } +} diff --git a/src/hashes/fast/xxh3/dispatch_tables.rs b/src/hashes/fast/xxh3/dispatch_tables.rs index 861e2838..c600d01b 100644 --- a/src/hashes/fast/xxh3/dispatch_tables.rs +++ b/src/hashes/fast/xxh3/dispatch_tables.rs @@ -5,105 +5,36 @@ pub use super::kernels::Xxh3KernelId as KernelId; use crate::platform::Caps; -#[cfg(any(test, feature = "diag"))] -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; - #[derive(Clone, Copy, Debug)] pub struct DispatchTable { - #[cfg(any(test, feature = "diag"))] - pub boundaries: [usize; 3], - #[cfg(any(test, feature = "diag"))] - pub xs: KernelId, - #[cfg(any(test, feature = "diag"))] - pub s: KernelId, - #[cfg(any(test, feature = "diag"))] - pub m: KernelId, - pub l: KernelId, + pub long: KernelId, } pub static DEFAULT_TABLE: DispatchTable = DispatchTable { - #[cfg(any(test, feature = "diag"))] - boundaries: DEFAULT_BOUNDARIES, - #[cfg(any(test, feature = "diag"))] - xs: KernelId::Portable, - #[cfg(any(test, feature = "diag"))] - s: KernelId::Portable, - #[cfg(any(test, feature = "diag"))] - m: KernelId::Portable, - l: KernelId::Portable, + long: KernelId::Portable, }; // Platform-specific tables /// x86-64 with AVX-512F: single-iteration per stripe. #[cfg(target_arch = "x86_64")] -pub static AVX512_TABLE: DispatchTable = DispatchTable { - #[cfg(any(test, feature = "diag"))] - boundaries: DEFAULT_BOUNDARIES, - #[cfg(any(test, feature = "diag"))] - xs: KernelId::Avx512, - #[cfg(any(test, feature = "diag"))] - s: KernelId::Avx512, - #[cfg(any(test, feature = "diag"))] - m: KernelId::Avx512, - l: KernelId::Avx512, -}; +pub static AVX512_TABLE: DispatchTable = DispatchTable { long: KernelId::Avx512 }; /// x86-64 with AVX2 (no AVX-512): two iterations per stripe. #[cfg(target_arch = "x86_64")] -pub static AVX2_TABLE: DispatchTable = DispatchTable { - #[cfg(any(test, feature = "diag"))] - boundaries: DEFAULT_BOUNDARIES, - #[cfg(any(test, feature = "diag"))] - xs: KernelId::Avx2, - #[cfg(any(test, feature = "diag"))] - s: KernelId::Avx2, - #[cfg(any(test, feature = "diag"))] - m: KernelId::Avx2, - l: KernelId::Avx2, -}; +pub static AVX2_TABLE: DispatchTable = DispatchTable { long: KernelId::Avx2 }; /// aarch64 with NEON: four iterations per stripe. #[cfg(target_arch = "aarch64")] -pub static NEON_TABLE: DispatchTable = DispatchTable { - #[cfg(any(test, feature = "diag"))] - boundaries: DEFAULT_BOUNDARIES, - #[cfg(any(test, feature = "diag"))] - xs: KernelId::Neon, - #[cfg(any(test, feature = "diag"))] - s: KernelId::Neon, - #[cfg(any(test, feature = "diag"))] - m: KernelId::Neon, - l: KernelId::Neon, -}; +pub static NEON_TABLE: DispatchTable = DispatchTable { long: KernelId::Neon }; /// POWER8+ with VSX: four iterations per stripe (128-bit vectors). #[cfg(all(target_arch = "powerpc64", target_endian = "little"))] -pub static VSX_TABLE: DispatchTable = DispatchTable { - #[cfg(any(test, feature = "diag"))] - boundaries: DEFAULT_BOUNDARIES, - #[cfg(any(test, feature = "diag"))] - xs: KernelId::Vsx, - #[cfg(any(test, feature = "diag"))] - s: KernelId::Vsx, - #[cfg(any(test, feature = "diag"))] - m: KernelId::Vsx, - l: KernelId::Vsx, -}; +pub static VSX_TABLE: DispatchTable = DispatchTable { long: KernelId::Vsx }; /// s390x z13+ with z/Vector: four iterations per stripe (128-bit vectors). #[cfg(target_arch = "s390x")] -pub static ZVECTOR_TABLE: DispatchTable = DispatchTable { - #[cfg(any(test, feature = "diag"))] - boundaries: DEFAULT_BOUNDARIES, - #[cfg(any(test, feature = "diag"))] - xs: KernelId::Vector, - #[cfg(any(test, feature = "diag"))] - s: KernelId::Vector, - #[cfg(any(test, feature = "diag"))] - m: KernelId::Vector, - l: KernelId::Vector, -}; +pub static ZVECTOR_TABLE: DispatchTable = DispatchTable { long: KernelId::Vector }; #[inline] #[must_use] @@ -130,7 +61,7 @@ pub fn select_runtime_table(caps: Caps) -> &'static DispatchTable { #[cfg(all(target_arch = "powerpc64", target_endian = "little"))] { - if caps.has(crate::platform::caps::power::POWER8_VECTOR) { + if caps.has(super::kernels::required_caps(KernelId::Vsx)) { return &VSX_TABLE; } } @@ -147,3 +78,21 @@ pub fn select_runtime_table(caps: Caps) -> &'static DispatchTable { &DEFAULT_TABLE } + +#[cfg(all(test, target_arch = "powerpc64", target_endian = "little"))] +mod tests { + use super::*; + use crate::platform::caps::power; + + #[test] + fn power_table_requires_every_target_feature() { + let required = power::ALTIVEC | power::VSX | power::POWER8_VECTOR; + assert_eq!(select_runtime_table(required).long, KernelId::Vsx); + for missing in [power::ALTIVEC, power::VSX, power::POWER8_VECTOR] { + assert_eq!( + select_runtime_table(required.difference(missing)).long, + KernelId::Portable + ); + } + } +} diff --git a/src/hashes/fast/xxh3/kernels.rs b/src/hashes/fast/xxh3/kernels.rs index c94026e8..99f30202 100644 --- a/src/hashes/fast/xxh3/kernels.rs +++ b/src/hashes/fast/xxh3/kernels.rs @@ -2,7 +2,7 @@ use super::{ACC_NB, DEFAULT_SECRET_SIZE}; use crate::platform::Caps; pub type StreamAccumulateFn = - fn([u64; ACC_NB], &[u8], usize, &[u8; DEFAULT_SECRET_SIZE], usize, usize, bool) -> [u64; ACC_NB]; + unsafe fn([u64; ACC_NB], &[u8], usize, &[u8; DEFAULT_SECRET_SIZE], usize, usize, bool) -> [u64; ACC_NB]; #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] @@ -41,24 +41,6 @@ impl Xxh3KernelId { } } -#[cfg(any(test, feature = "diag"))] -#[must_use] -pub fn hash64_fn(id: Xxh3KernelId) -> fn(&[u8], u64) -> u64 { - match id { - Xxh3KernelId::Portable => super::xxh3_64_with_seed, - #[cfg(target_arch = "x86_64")] - Xxh3KernelId::Avx2 => super::x86_64_avx2::xxh3_64_with_seed, - #[cfg(target_arch = "aarch64")] - Xxh3KernelId::Neon => super::aarch64_neon::xxh3_64_with_seed, - #[cfg(target_arch = "x86_64")] - Xxh3KernelId::Avx512 => super::x86_64_avx512::xxh3_64_with_seed, - #[cfg(all(target_arch = "powerpc64", target_endian = "little"))] - Xxh3KernelId::Vsx => super::power::xxh3_64_with_seed, - #[cfg(target_arch = "s390x")] - Xxh3KernelId::Vector => super::s390x::xxh3_64_with_seed, - } -} - /// Long-path-only entry for 64-bit hash (>240B, no ≤240B length checks). #[must_use] pub fn hash64_long_fn(id: Xxh3KernelId) -> fn(&[u8], u64) -> u64 { @@ -124,7 +106,9 @@ pub const fn required_caps(id: Xxh3KernelId) -> Caps { #[cfg(target_arch = "x86_64")] Xxh3KernelId::Avx512 => crate::platform::caps::x86::AVX512F, #[cfg(all(target_arch = "powerpc64", target_endian = "little"))] - Xxh3KernelId::Vsx => crate::platform::caps::power::POWER8_VECTOR, + Xxh3KernelId::Vsx => crate::platform::caps::power::ALTIVEC + .union(crate::platform::caps::power::VSX) + .union(crate::platform::caps::power::POWER8_VECTOR), #[cfg(target_arch = "s390x")] Xxh3KernelId::Vector => crate::platform::caps::s390x::VECTOR, } diff --git a/src/hashes/fast/xxh3/power.rs b/src/hashes/fast/xxh3/power.rs index 6851a47a..bae33282 100644 --- a/src/hashes/fast/xxh3/power.rs +++ b/src/hashes/fast/xxh3/power.rs @@ -269,7 +269,12 @@ unsafe fn stream_accumulate_inner( } } -pub(crate) fn stream_accumulate( +/// # Safety +/// +/// The caller must verify AltiVec, VSX, and POWER8-vector support, pass +/// `stripes > 0`, and keep every requested input and secret stripe within its +/// slice. +pub(crate) unsafe fn stream_accumulate( initial: [u64; ACC_NB], input: &[u8], input_offset: usize, @@ -278,9 +283,8 @@ pub(crate) fn stream_accumulate( stripes: usize, scramble_after: bool, ) -> [u64; ACC_NB] { - // SAFETY: Calling the POWER8 vector streaming kernel because: - // 1. Runtime dispatch selects this wrapper only after POWER8 vector capability detection. - // 2. `stream_accumulate_inner` validates all slice offsets before pointer arithmetic. + // SAFETY: The caller contract provides the required POWER vector features + // and proves every raw input and secret load is in bounds. unsafe { stream_accumulate_inner( initial, @@ -368,23 +372,6 @@ pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { } } -/// XXH3 64-bit hash — POWER VSX kernel. -/// -/// Delegates ≤240 B to portable scalar paths; >240 B uses VSX accumulator. -#[cfg(any(test, feature = "diag"))] -pub fn xxh3_64_with_seed(input: &[u8], seed: u64) -> u64 { - if input.len() <= 16 { - return super::xxh3_64_0to16(input, seed, &DEFAULT_SECRET); - } - if input.len() <= 128 { - return super::xxh3_64_7to128(input, seed, &DEFAULT_SECRET); - } - if input.len() <= super::MID_SIZE_MAX { - return super::xxh3_64_129to240(input, seed, &DEFAULT_SECRET); - } - xxh3_64_long(input, seed) -} - /// Long-path entry point (>240B) — no ≤240B branches. pub fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { if seed == 0 { diff --git a/src/hashes/fast/xxh3/s390x.rs b/src/hashes/fast/xxh3/s390x.rs index 552f83bf..cb387664 100644 --- a/src/hashes/fast/xxh3/s390x.rs +++ b/src/hashes/fast/xxh3/s390x.rs @@ -330,7 +330,11 @@ unsafe fn stream_accumulate_inner( } } -pub(crate) fn stream_accumulate( +/// # Safety +/// +/// The caller must verify z/Vector support, pass `stripes > 0`, and keep every +/// requested input and secret stripe within its slice. +pub(crate) unsafe fn stream_accumulate( initial: [u64; ACC_NB], input: &[u8], input_offset: usize, @@ -339,9 +343,8 @@ pub(crate) fn stream_accumulate( stripes: usize, scramble_after: bool, ) -> [u64; ACC_NB] { - // SAFETY: Calling the z/Vector streaming kernel because: - // 1. Runtime dispatch selects this wrapper only after vector-facility detection. - // 2. `stream_accumulate_inner` validates all slice offsets before pointer arithmetic. + // SAFETY: The caller contract provides z/Vector support and proves every raw + // input and secret load is in bounds. unsafe { stream_accumulate_inner( initial, @@ -430,23 +433,6 @@ pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { } } -/// XXH3 64-bit hash — s390x z/Vector kernel. -/// -/// Delegates ≤240 B to portable scalar paths; >240 B uses z/Vector accumulator. -#[cfg(any(test, feature = "diag"))] -pub fn xxh3_64_with_seed(input: &[u8], seed: u64) -> u64 { - if input.len() <= 16 { - return super::xxh3_64_0to16(input, seed, &DEFAULT_SECRET); - } - if input.len() <= 128 { - return super::xxh3_64_7to128(input, seed, &DEFAULT_SECRET); - } - if input.len() <= super::MID_SIZE_MAX { - return super::xxh3_64_129to240(input, seed, &DEFAULT_SECRET); - } - xxh3_64_long(input, seed) -} - /// Long-path entry point (>240B) — no ≤240B branches. pub fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { if seed == 0 { diff --git a/src/hashes/fast/xxh3/stream.rs b/src/hashes/fast/xxh3/stream.rs index 053f678b..60f76da2 100644 --- a/src/hashes/fast/xxh3/stream.rs +++ b/src/hashes/fast/xxh3/stream.rs @@ -23,7 +23,8 @@ fn consume_stripes( input: &[u8], secret: &[u8; DEFAULT_SECRET_SIZE], ) -> ([u64; ACC_NB], usize) { - debug_assert!(count.strict_mul(STRIPE_LEN) <= input.len()); + assert!(count.strict_mul(STRIPE_LEN) <= input.len()); + assert!(accumulated < STRIPES_PER_BLOCK); let mut input_offset = 0usize; while count != 0 { let to_block_end = STRIPES_PER_BLOCK.strict_sub(accumulated); @@ -34,15 +35,22 @@ fn consume_stripes( } else { stream_accumulate }; - acc = kernel( - acc, - input, - input_offset, - secret, - accumulated.strict_mul(SECRET_CONSUME_RATE), - stripes, - scramble_after, - ); + // SAFETY: `count * STRIPE_LEN <= input.len()` is checked above and + // `input_offset` advances by exactly the consumed stripes. `accumulated` + // stays below `STRIPES_PER_BLOCK`, so the selected stripe range remains + // within the fixed secret. Dispatch selected `kernel` only after verifying + // its target-feature capability. + acc = unsafe { + kernel( + acc, + input, + input_offset, + secret, + accumulated.strict_mul(SECRET_CONSUME_RATE), + stripes, + scramble_after, + ) + }; input_offset = input_offset.strict_add(stripes.strict_mul(STRIPE_LEN)); count = count.strict_sub(stripes); if scramble_after { @@ -542,9 +550,8 @@ mod tests { fn streaming_simd_kernels_match_portable_accumulator() { use super::super::kernels::{Xxh3KernelId, required_caps, stream_accumulate_fn}; - let input = data(INTERNAL_BUFFER_SIZE); + let input = data(INTERNAL_BUFFER_SIZE * 4 + 64); let secret = custom_default_secret(42); - let expected = stream_accumulate_fn(Xxh3KernelId::Portable)(INITIAL_ACC, &input, 0, &secret, 8, 4, true); #[cfg(target_arch = "x86_64")] let kernels = &[Xxh3KernelId::Avx2, Xxh3KernelId::Avx512][..]; #[cfg(target_arch = "aarch64")] @@ -564,11 +571,122 @@ mod tests { let caps = crate::platform::caps(); for &kernel in kernels { if caps.has(required_caps(kernel)) { - assert_eq!( - stream_accumulate_fn(kernel)(INITIAL_ACC, &input, 0, &secret, 8, 4, true), - expected - ); + for offset in 0..64 { + for scramble_after in [false, true] { + // SAFETY: One stripe starting at `offset` is inside `input`; the + // zero secret offset leaves a full stripe inside `secret`; runtime + // capabilities satisfy the selected kernel's target features. + let expected = unsafe { + stream_accumulate_fn(Xxh3KernelId::Portable)(INITIAL_ACC, &input, offset, &secret, 0, 1, scramble_after) + }; + // SAFETY: Same bounds as the portable call above, and + // `required_caps(kernel)` was checked before entering the loop. + let actual = + unsafe { stream_accumulate_fn(kernel)(INITIAL_ACC, &input, offset, &secret, 0, 1, scramble_after) }; + assert_eq!(actual, expected, "kernel={kernel:?} offset={offset}"); + } + } + + for (input_offset, secret_offset, stripes, scramble_after) in + [(0, 0, 16, true), (1, 8, 4, false), (31, 120, 1, true)] + { + // SAFETY: Every table row keeps all input stripes within `input` and + // all secret stripes within `secret`. + let expected = unsafe { + stream_accumulate_fn(Xxh3KernelId::Portable)( + INITIAL_ACC, + &input, + input_offset, + &secret, + secret_offset, + stripes, + scramble_after, + ) + }; + // SAFETY: The table bounds are proved above and runtime capabilities + // satisfy the selected kernel's target features. + let actual = unsafe { + stream_accumulate_fn(kernel)( + INITIAL_ACC, + &input, + input_offset, + &secret, + secret_offset, + stripes, + scramble_after, + ) + }; + assert_eq!( + actual, expected, + "kernel={kernel:?} input_offset={input_offset} secret_offset={secret_offset} stripes={stripes}" + ); + } } } } + + #[test] + #[cfg(not(miri))] + fn available_long_kernels_match_independent_oracle_across_alignments_and_tails() { + use super::super::kernels::{Xxh3KernelId, hash64_long_fn, hash128_long_fn, required_caps}; + + #[cfg(target_arch = "x86_64")] + let kernels = &[Xxh3KernelId::Avx2, Xxh3KernelId::Avx512][..]; + #[cfg(target_arch = "aarch64")] + let kernels = &[Xxh3KernelId::Neon][..]; + #[cfg(all(target_arch = "powerpc64", target_endian = "little"))] + let kernels = &[Xxh3KernelId::Vsx][..]; + #[cfg(target_arch = "s390x")] + let kernels = &[Xxh3KernelId::Vector][..]; + #[cfg(not(any( + target_arch = "x86_64", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "s390x" + )))] + let kernels = &[][..]; + + let input = data(2113); + let caps = crate::platform::caps(); + for &kernel in kernels { + if !caps.has(required_caps(kernel)) { + continue; + } + let hash64 = hash64_long_fn(kernel); + let hash128 = hash128_long_fn(kernel); + for seed in [0, 0x243f_6a88_85a3_08d3] { + for offset in 0..64 { + for len in [241, 255, 256, 257, 1023, 1024, 1025, 2047, 2048, 2049] { + let data = &input[offset..offset + len]; + assert_eq!( + hash64(data, seed), + xxhash_rust::xxh3::xxh3_64_with_seed(data, seed), + "64-bit kernel={kernel:?} seed={seed:#x} offset={offset} len={len}" + ); + assert_eq!( + hash128(data, seed), + xxhash_rust::xxh3::xxh3_128_with_seed(data, seed), + "128-bit kernel={kernel:?} seed={seed:#x} offset={offset} len={len}" + ); + } + } + } + } + } + + #[test] + #[should_panic] + fn consume_stripes_rejects_short_input_before_kernel_call() { + use super::super::kernels::{Xxh3KernelId, stream_accumulate_fn}; + + let secret = custom_default_secret(0); + consume_stripes( + stream_accumulate_fn(Xxh3KernelId::Portable), + INITIAL_ACC, + 1, + 0, + &[], + &secret, + ); + } } diff --git a/src/hashes/fast/xxh3/x86_64_avx2.rs b/src/hashes/fast/xxh3/x86_64_avx2.rs index 8384b711..42e2eedf 100644 --- a/src/hashes/fast/xxh3/x86_64_avx2.rs +++ b/src/hashes/fast/xxh3/x86_64_avx2.rs @@ -166,7 +166,11 @@ unsafe fn stream_accumulate_inner( } } -pub(crate) fn stream_accumulate( +/// # Safety +/// +/// The caller must verify AVX2 support, pass `stripes > 0`, and keep every +/// requested input and secret stripe within its slice. +pub(crate) unsafe fn stream_accumulate( initial: [u64; ACC_NB], input: &[u8], input_offset: usize, @@ -175,9 +179,8 @@ pub(crate) fn stream_accumulate( stripes: usize, scramble_after: bool, ) -> [u64; ACC_NB] { - // SAFETY: Calling the AVX2 streaming kernel because: - // 1. Runtime dispatch selects this wrapper only after AVX2 capability detection. - // 2. `stream_accumulate_inner` validates all slice offsets before pointer arithmetic. + // SAFETY: The caller contract provides AVX2 support and proves every raw + // input and secret load is in bounds. unsafe { stream_accumulate_inner( initial, @@ -269,23 +272,6 @@ pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { } } -/// XXH3 64-bit hash — AVX2 kernel. -/// -/// Delegates ≤240 B to portable scalar paths; >240 B uses AVX2 accumulator. -#[cfg(any(test, feature = "diag"))] -pub fn xxh3_64_with_seed(input: &[u8], seed: u64) -> u64 { - if input.len() <= 16 { - return super::xxh3_64_0to16(input, seed, &DEFAULT_SECRET); - } - if input.len() <= 128 { - return super::xxh3_64_7to128(input, seed, &DEFAULT_SECRET); - } - if input.len() <= super::MID_SIZE_MAX { - return super::xxh3_64_129to240(input, seed, &DEFAULT_SECRET); - } - xxh3_64_long(input, seed) -} - /// Long-path entry point (>240B) — no ≤240B branches. pub fn xxh3_128_long_default(input: &[u8]) -> u128 { // SAFETY: Dispatcher verifies AVX2 before selecting this kernel. diff --git a/src/hashes/fast/xxh3/x86_64_avx512.rs b/src/hashes/fast/xxh3/x86_64_avx512.rs index 47f60ecf..13a95ac5 100644 --- a/src/hashes/fast/xxh3/x86_64_avx512.rs +++ b/src/hashes/fast/xxh3/x86_64_avx512.rs @@ -141,7 +141,11 @@ unsafe fn stream_accumulate_inner( } } -pub(crate) fn stream_accumulate( +/// # Safety +/// +/// The caller must verify AVX-512F support, pass `stripes > 0`, and keep every +/// requested input and secret stripe within its slice. +pub(crate) unsafe fn stream_accumulate( initial: [u64; ACC_NB], input: &[u8], input_offset: usize, @@ -150,9 +154,8 @@ pub(crate) fn stream_accumulate( stripes: usize, scramble_after: bool, ) -> [u64; ACC_NB] { - // SAFETY: Calling the AVX-512F streaming kernel because: - // 1. Runtime dispatch selects this wrapper only after AVX-512F capability detection. - // 2. `stream_accumulate_inner` validates all slice offsets before pointer arithmetic. + // SAFETY: The caller contract provides AVX-512F support and proves every raw + // input and secret load is in bounds. unsafe { stream_accumulate_inner( initial, @@ -245,23 +248,6 @@ pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { } } -/// XXH3 64-bit hash — AVX-512 kernel. -/// -/// Delegates ≤240 B to portable scalar paths; >240 B uses AVX-512 accumulator. -#[cfg(any(test, feature = "diag"))] -pub fn xxh3_64_with_seed(input: &[u8], seed: u64) -> u64 { - if input.len() <= 16 { - return super::xxh3_64_0to16(input, seed, &DEFAULT_SECRET); - } - if input.len() <= 128 { - return super::xxh3_64_7to128(input, seed, &DEFAULT_SECRET); - } - if input.len() <= super::MID_SIZE_MAX { - return super::xxh3_64_129to240(input, seed, &DEFAULT_SECRET); - } - xxh3_64_long(input, seed) -} - /// Long-path entry point (>240B) — no ≤240B branches. pub fn xxh3_128_long_default(input: &[u8]) -> u128 { // SAFETY: Dispatcher verifies AVX-512F before selecting this kernel. diff --git a/src/hashes/introspect.rs b/src/hashes/introspect.rs index f941977d..99d74b81 100644 --- a/src/hashes/introspect.rs +++ b/src/hashes/introspect.rs @@ -179,12 +179,12 @@ impl_hash_kernel_introspect!( #[cfg(feature = "xxh3")] impl_hash_kernel_introspect!( crate::hashes::fast::xxh3::Xxh3_64, - crate::hashes::fast::xxh3::dispatch::kernel_name_for_len + crate::hashes::fast::xxh3::dispatch::kernel_name64_for_len ); #[cfg(feature = "xxh3")] impl_hash_kernel_introspect!( crate::hashes::fast::xxh3::Xxh3_128, - crate::hashes::fast::xxh3::dispatch::kernel_name_for_len + crate::hashes::fast::xxh3::dispatch::kernel_name128_for_len ); #[cfg(test)] mod tests { diff --git a/src/hashes/io.rs b/src/hashes/io.rs index 5303026d..f61367e3 100644 --- a/src/hashes/io.rs +++ b/src/hashes/io.rs @@ -4,11 +4,9 @@ //! [`std::io::Read`] and [`std::io::Write`] implementations to compute digests //! transparently during I/O operations. //! -//! # Performance -//! -//! - Zero-cost abstraction: All methods are `#[inline]` -//! - Vectored I/O support: Uses the `update_vectored` method when available -//! - Correctness: Only hashes bytes actually transferred (handles short reads/writes) +//! Vectored adapters update the digest once for each transferred slice. All +//! adapters hash only bytes actually transferred, including short reads and +//! writes. //! //! # Example //! diff --git a/src/hashes/mod.rs b/src/hashes/mod.rs index 3d49ef47..4248c89e 100644 --- a/src/hashes/mod.rs +++ b/src/hashes/mod.rs @@ -45,16 +45,16 @@ //! ```toml //! [dependencies] //! # Smallest SHA-2-only build -//! rscrypto = { version = "0.5.0", default-features = false, features = ["sha2"] } +//! rscrypto = { version = "0.7", default-features = false, features = ["sha2"] } //! //! # All cryptographic hashes -//! rscrypto = { version = "0.5.0", default-features = false, features = ["crypto-hashes"] } +//! rscrypto = { version = "0.7", default-features = false, features = ["crypto-hashes"] } //! //! # Fast non-cryptographic hashes only -//! rscrypto = { version = "0.5.0", default-features = false, features = ["fast-hashes"] } +//! rscrypto = { version = "0.7", default-features = false, features = ["fast-hashes"] } //! //! # Everything hash-related -//! rscrypto = { version = "0.5.0", default-features = false, features = ["hashes"] } +//! rscrypto = { version = "0.7", default-features = false, features = ["hashes"] } //! ``` //! //! # API Conventions @@ -62,7 +62,8 @@ //! - Fixed-output digests use `Type::digest(data)` for one-shot and `new` / `update` / `finalize` / //! `reset` for streaming. //! - XOFs use `Type::xof(data)` for one-shot and `finalize_xof()` for streaming squeeze readers. -//! - Fast hashes are one-shot only and implement [`crate::traits::FastHash`]. +//! - Fast hashes implement the one-shot [`crate::traits::FastHash`] trait and also expose +//! algorithm-specific streaming hashers. //! //! # Modules //! diff --git a/src/platform/caps.rs b/src/platform/caps.rs index e80848de..c96bcb84 100644 --- a/src/platform/caps.rs +++ b/src/platform/caps.rs @@ -406,7 +406,7 @@ pub mod x86 { // ─── Vendor Identification ─── /// AMD CPU vendor flag (set when CPUID reports "AuthenticAMD"). - /// Used for vendor-aware dispatch where optimal kernel differs by vendor. + /// Used for vendor-aware dispatch policy. pub const AMD: Caps = Caps::bit(49); /// AMD Zen 5+ (CPUID family ≥ 0x1A). The 6-wide dispatch pipeline favors @@ -414,8 +414,8 @@ pub mod x86 { pub const AMD_ZEN5: Caps = Caps::bit(50); /// Intel Sapphire Rapids Xeon (CPUID family 6, model 0x8F). - /// Used for microarchitecture-specific dispatch fixes when feature bits alone - /// do not explain a measured regression. + /// Used when dispatch policy needs a microarchitecture discriminator beyond + /// architectural feature bits. pub const INTEL_SAPPHIRE_RAPIDS: Caps = Caps::bit(51); // ─── Combined Capability Masks ─── @@ -452,12 +452,6 @@ pub mod x86 { /// The SSE4.2 `crc32` instruction is polynomial-locked to CRC-32C (Castagnoli, /// 0x1EDC6F41). It cannot compute CRC-32 IEEE (0x04C11DB7). /// - /// # Performance - /// - /// - Throughput: ~20 GB/s (faster than PCLMULQDQ for small buffers) - /// - Latency: 3 cycles - /// - Available on all modern x86_64 CPUs since Nehalem (2008) - /// /// For CRC-32 IEEE, use PCLMULQDQ-based algorithms instead. pub const CRC32C_READY: Caps = Caps([SSE42.0[0], 0, 0, 0]); } @@ -550,7 +544,7 @@ pub mod aarch64 { /// PMULL-ready: polynomial multiply long (PMULL). pub const PMULL_READY: Caps = PMULL; - /// PMULL+EOR3-ready: PMULL + SHA3 (SHA3 provides EOR3 for faster GHASH). + /// PMULL+EOR3-ready: PMULL + SHA3 (SHA3 provides three-operand EOR3). pub const PMULL_EOR3_READY: Caps = PMULL.union(SHA3); /// CRC32C-ready: CRC extension diff --git a/src/platform/detect.rs b/src/platform/detect.rs index 9e360971..278c3dec 100644 --- a/src/platform/detect.rs +++ b/src/platform/detect.rs @@ -80,12 +80,14 @@ impl Detected { #[inline] #[must_use] + #[cfg(any(feature = "std", all(not(feature = "std"), target_has_atomic = "64")))] const fn is_portable(self) -> bool { self.caps.is_empty() && matches!(self.arch, Arch::Other) } } #[cold] +#[cfg(any(feature = "std", all(not(feature = "std"), target_has_atomic = "64")))] fn validate_override(value: Option) -> Result, OverrideError> { let Some(det) = value else { return Ok(None); diff --git a/src/platform/detect/arch/aarch64.rs b/src/platform/detect/arch/aarch64.rs index fcfd007d..c3011446 100644 --- a/src/platform/detect/arch/aarch64.rs +++ b/src/platform/detect/arch/aarch64.rs @@ -17,7 +17,6 @@ fn detect_aarch64() -> Detected { /// Batch extraction of aarch64 features from /proc/self/auxv. /// /// Reads AT_HWCAP and AT_HWCAP2 once from the ELF auxiliary vector. -/// This is faster than calling is_aarch64_feature_detected! 20+ times. /// Pure Rust - no libc dependency. /// /// Works on Linux and Android (both use procfs with ELF auxv format). @@ -612,24 +611,6 @@ fn detect_apple_sme_features() -> Caps { caps |= aarch64::SME_F16F16; } - // ─── Fallback: Infer SME from chip generation if sysctl unavailable ─── - // This handles cases where the OS doesn't expose SME sysctl keys yet. - // M4 has SME, M5 has SME2p1 + additional features. - if caps.is_empty() - && let Some(chip_gen) = detect_apple_silicon_gen() - { - match chip_gen { - AppleSiliconGen::M4 => { - caps |= aarch64::SME; - } - AppleSiliconGen::M5 => { - // M5 has SME2p1, SMEB16B16, SMEF16F16 per LLVM - caps |= aarch64::SME | aarch64::SME2 | aarch64::SME2P1 | aarch64::SME_B16B16 | aarch64::SME_F16F16; - } - _ => {} - } - } - caps } diff --git a/src/platform/detect/arch/power.rs b/src/platform/detect/arch/power.rs index b343a4b7..4e8c685f 100644 --- a/src/platform/detect/arch/power.rs +++ b/src/platform/detect/arch/power.rs @@ -28,21 +28,10 @@ fn detect_power() -> Detected { fn runtime_power() -> Caps { use std::{fs::File, io::Read}; - use crate::platform::caps::power; - // ELF auxiliary vector entry types const AT_HWCAP: u64 = 16; const AT_HWCAP2: u64 = 26; - // HWCAP masks (from linux/arch/powerpc/include/uapi/asm/cputable.h) - const PPC_FEATURE_HAS_ALTIVEC: u64 = 0x1000_0000; - const PPC_FEATURE_HAS_VSX: u64 = 0x0000_0080; - - // HWCAP2 masks - const PPC_FEATURE2_ARCH_2_07: u64 = 0x8000_0000; // POWER8 ISA (v2.07) - const PPC_FEATURE2_ARCH_3_00: u64 = 0x0080_0000; // POWER9 ISA (v3.00) - const PPC_FEATURE2_ARCH_3_1: u64 = 0x0004_0000; // POWER10 ISA (v3.1) - let (hwcap, hwcap2) = (|| -> Option<(u64, u64)> { let mut file = File::open("/proc/self/auxv").ok()?; let mut buf = [0u8; 4096]; @@ -68,6 +57,24 @@ fn runtime_power() -> Caps { })() .unwrap_or((0, 0)); + decode_power_hwcap(hwcap, hwcap2) +} + +#[cfg(all( + target_arch = "powerpc64", + feature = "std", + any(target_os = "linux", target_os = "android") +))] +fn decode_power_hwcap(hwcap: u64, hwcap2: u64) -> Caps { + use crate::platform::caps::power; + + // linux/arch/powerpc/include/uapi/asm/cputable.h + const PPC_FEATURE_HAS_ALTIVEC: u64 = 0x1000_0000; + const PPC_FEATURE_HAS_VSX: u64 = 0x0000_0080; + const PPC_FEATURE2_ARCH_2_07: u64 = 0x8000_0000; + const PPC_FEATURE2_ARCH_3_00: u64 = 0x0080_0000; + const PPC_FEATURE2_ARCH_3_1: u64 = 0x0004_0000; + let mut caps = Caps::NONE; if hwcap & PPC_FEATURE_HAS_ALTIVEC != 0 { @@ -86,9 +93,50 @@ fn runtime_power() -> Caps { caps |= power::POWER8_VECTOR | power::POWER8_CRYPTO; } + if hwcap2 & (PPC_FEATURE2_ARCH_2_07 | PPC_FEATURE2_ARCH_3_00 | PPC_FEATURE2_ARCH_3_1) != 0 { + caps |= power::QUADWORD_ATOMICS | power::PARTWORD_ATOMICS; + } + caps } +#[cfg(all( + test, + target_arch = "powerpc64", + feature = "std", + any(target_os = "linux", target_os = "android") +))] +mod power_tests { + use super::*; + use crate::platform::caps::power; + + #[test] + fn power_isa_levels_include_their_predecessors_and_atomics() { + const ARCH_2_07: u64 = 0x8000_0000; + const ARCH_3_00: u64 = 0x0080_0000; + const ARCH_3_1: u64 = 0x0004_0000; + const ATOMICS: Caps = power::QUADWORD_ATOMICS.union(power::PARTWORD_ATOMICS); + + assert_eq!(decode_power_hwcap(0, 0), Caps::NONE); + assert_eq!( + decode_power_hwcap(0, ARCH_2_07), + power::POWER8_VECTOR | power::POWER8_CRYPTO | ATOMICS + ); + assert_eq!( + decode_power_hwcap(0, ARCH_3_00), + power::POWER9_VECTOR | power::POWER8_VECTOR | power::POWER8_CRYPTO | ATOMICS + ); + assert_eq!( + decode_power_hwcap(0, ARCH_3_1), + power::POWER10_VECTOR + | power::POWER9_VECTOR + | power::POWER8_VECTOR + | power::POWER8_CRYPTO + | ATOMICS + ); + } +} + /// Runtime Power detection for other platforms. #[cfg(all( target_arch = "powerpc64", diff --git a/src/platform/detect/arch/x86.rs b/src/platform/detect/arch/x86.rs index d74689c7..17835c4e 100644 --- a/src/platform/detect/arch/x86.rs +++ b/src/platform/detect/arch/x86.rs @@ -7,15 +7,34 @@ fn detect_x86_64() -> Detected { // Runtime detection extracts features + vendor/family/model in batch #[cfg(feature = "std")] - let (runtime_caps, is_amd, family, model) = { + let (runtime_caps, is_amd, family, model, amx_permission) = { let batch = cpuid_batch_x86_64(); - (batch.caps, batch.is_amd, batch.family, batch.model) + ( + batch.caps, + batch.is_amd, + batch.family, + batch.model, + batch.amx_permission, + ) }; #[cfg(feature = "std")] let mut caps = caps_static.union(runtime_caps); #[cfg(not(feature = "std"))] - let caps = caps_static; + let mut caps = caps_static; + + #[cfg(feature = "std")] + { + caps = gate_x86_amx_permission(caps, amx_permission); + } + + #[cfg(all( + not(feature = "std"), + any(target_os = "linux", target_os = "android") + ))] + { + caps = gate_x86_amx_permission(caps, false); + } // Hybrid Intel AVX-512 Safety: Clear AVX-512 caps on hybrid CPUs // On hybrid Intel CPUs (Alder Lake, Raptor Lake, etc.), the P-cores have @@ -43,9 +62,7 @@ fn detect_x86_64() -> Detected { .difference(x86::AVX512VPOPCNTDQ) .difference(x86::AVX512BF16) .difference(x86::AVX512FP16) - .difference(x86::VPCLMULQDQ) - .difference(x86::VAES) - .difference(x86::GFNI) + .difference(x86::AVX512VP2INTERSECT) .difference(x86::AVX10_1) .difference(x86::AVX10_2); } @@ -57,6 +74,23 @@ fn detect_x86_64() -> Detected { } } +#[cfg(target_arch = "x86_64")] +const X86_ALL_AMX: Caps = crate::platform::caps::x86::AMX_TILE + .union(crate::platform::caps::x86::AMX_BF16) + .union(crate::platform::caps::x86::AMX_INT8) + .union(crate::platform::caps::x86::AMX_FP16) + .union(crate::platform::caps::x86::AMX_COMPLEX); + +#[cfg(target_arch = "x86_64")] +#[inline] +const fn gate_x86_amx_permission(caps: Caps, permitted: bool) -> Caps { + if permitted { + caps + } else { + caps.difference(X86_ALL_AMX) + } +} + #[cfg(target_arch = "x86")] fn detect_x86() -> Detected { // Start with compile-time detected features @@ -128,6 +162,42 @@ struct CpuidBatch { is_amd: bool, family: u32, model: u32, + amx_permission: bool, +} + +#[cfg(all(target_arch = "x86_64", feature = "std"))] +#[derive(Clone, Copy, Default)] +struct CpuidRegisters { + eax: u32, + ebx: u32, + ecx: u32, + edx: u32, +} + +#[cfg(all(target_arch = "x86_64", feature = "std"))] +impl From for CpuidRegisters { + fn from(result: core::arch::x86_64::CpuidResult) -> Self { + Self { + eax: result.eax, + ebx: result.ebx, + ecx: result.ecx, + edx: result.edx, + } + } +} + +#[cfg(all(target_arch = "x86_64", feature = "std"))] +#[derive(Clone, Copy, Default)] +struct CpuidSnapshot { + leaf0: CpuidRegisters, + leaf1: CpuidRegisters, + leaf7_0: CpuidRegisters, + leaf7_1: CpuidRegisters, + leaf24_0: CpuidRegisters, + extended_leaf0: CpuidRegisters, + extended_leaf1: CpuidRegisters, + xcr0: u64, + amx_permission: bool, } /// Batch CPUID extraction - extracts all features and CPU info in minimal CPUID calls. @@ -137,8 +207,7 @@ struct CpuidBatch { /// - Leaf 1: processor info + basic features /// - Leaf 7.0: extended features /// - Leaf 7.1: more extended features -/// - Leaf 0x24: AVX10 detection (if max leaf >= 0x24) -/// - Leaf 0x29: APX detection (if max leaf >= 0x29) +/// - Leaf 0x24: AVX10 version (if leaf 7 reports AVX10 and max leaf permits it) /// - Leaf 0x80000001: AMD-specific features /// /// **Critical**: This function properly gates AVX/AVX-512 features by checking @@ -152,26 +221,140 @@ struct CpuidBatch { fn cpuid_batch_x86_64() -> CpuidBatch { use core::arch::x86_64::_xgetbv; + let leaf0 = CpuidRegisters::from(cpuid_leaf(0)); + let leaf1 = if leaf0.eax >= 1 { + CpuidRegisters::from(cpuid_leaf(1)) + } else { + CpuidRegisters::default() + }; + let leaf7_0 = if leaf0.eax >= 7 { + CpuidRegisters::from(cpuid_leaf_count(7, 0)) + } else { + CpuidRegisters::default() + }; + let leaf7_1 = if leaf0.eax >= 7 && leaf7_0.eax >= 1 { + CpuidRegisters::from(cpuid_leaf_count(7, 1)) + } else { + CpuidRegisters::default() + }; + let leaf24_0 = if leaf0.eax >= 0x24 && leaf7_1.edx & (1 << 19) != 0 { + CpuidRegisters::from(cpuid_leaf_count(0x24, 0)) + } else { + CpuidRegisters::default() + }; + let extended_leaf0 = CpuidRegisters::from(cpuid_leaf(0x8000_0000)); + let extended_leaf1 = if extended_leaf0.eax >= 0x8000_0001 { + CpuidRegisters::from(cpuid_leaf(0x8000_0001)) + } else { + CpuidRegisters::default() + }; + let xcr0 = if leaf1.ecx & (1 << 27) != 0 { + // SAFETY: leaf 1 reported OSXSAVE, so XGETBV is enabled; index zero reads + // the architectural extended-state mask without accessing Rust memory. + unsafe { _xgetbv(0) } + } else { + 0 + }; + + decode_cpuid_x86_64(CpuidSnapshot { + leaf0, + leaf1, + leaf7_0, + leaf7_1, + leaf24_0, + extended_leaf0, + extended_leaf1, + xcr0, + amx_permission: amx_xstate_permission_x86_64(), + }) +} + +#[cfg(all( + target_arch = "x86_64", + feature = "std", + any(target_os = "linux", target_os = "android") +))] +#[allow(unsafe_code)] +fn amx_xstate_permission_x86_64() -> bool { + const SYS_ARCH_PRCTL: isize = 158; + const ARCH_GET_XCOMP_PERM: usize = 0x1022; + const XCOMP_TILE_MASK: u64 = (1 << 17) | (1 << 18); + + let mut permissions = 0u64; + let mut result = SYS_ARCH_PRCTL; + + // SAFETY: Linux x86_64 syscall ABI invocation because: + // 1. syscall 158 is arch_prctl on the x86_64 Linux ABI. + // 2. ARCH_GET_XCOMP_PERM writes one u64 through the valid exclusive pointer + // in RSI and does not retain it. + // 3. syscall clobbers RAX, RCX, and R11; all are declared, and no Rust + // reference is live across an undeclared register or stack mutation. + unsafe { + core::arch::asm!( + "syscall", + inlateout("rax") result, + in("rdi") ARCH_GET_XCOMP_PERM, + in("rsi") &mut permissions, + lateout("rcx") _, + lateout("r11") _, + options(nostack), + ); + } + + result == 0 && permissions & XCOMP_TILE_MASK == XCOMP_TILE_MASK +} + +#[cfg(all( + target_arch = "x86_64", + feature = "std", + not(any(target_os = "linux", target_os = "android")) +))] +const fn amx_xstate_permission_x86_64() -> bool { + true +} + +#[cfg(all(target_arch = "x86_64", feature = "std"))] +fn decode_cpuid_x86_64(snapshot: CpuidSnapshot) -> CpuidBatch { use crate::platform::caps::x86; - // XCR0 bit masks for OS support verification - // Bits 1-2: XMM (SSE) + YMM (AVX) state - must be set for AVX const XCR0_AVX_MASK: u64 = 0x6; - // Bits 5-7: opmask + ZMM_Hi256 + Hi16_ZMM state - must be set for AVX-512 const XCR0_AVX512_MASK: u64 = 0xE0; + const XCR0_AMX_MASK: u64 = (1 << 17) | (1 << 18); + const XCR0_APX_MASK: u64 = 1 << 19; let mut caps = Caps::NONE; + let cpuid0 = snapshot.leaf0; + let cpuid1 = if cpuid0.eax >= 1 { + snapshot.leaf1 + } else { + CpuidRegisters::default() + }; + let cpuid7 = if cpuid0.eax >= 7 { + snapshot.leaf7_0 + } else { + CpuidRegisters::default() + }; + let cpuid7_1 = if cpuid0.eax >= 7 && cpuid7.eax >= 1 { + snapshot.leaf7_1 + } else { + CpuidRegisters::default() + }; + let cpuid24 = if cpuid0.eax >= 0x24 { + snapshot.leaf24_0 + } else { + CpuidRegisters::default() + }; + let cpuid_ext = if snapshot.extended_leaf0.eax >= 0x8000_0001 { + snapshot.extended_leaf1 + } else { + CpuidRegisters::default() + }; - // CPUID leaf 0: vendor string - let cpuid0 = cpuid_leaf(0); // "GenuineIntel" has EBX/EDX/ECX = "Genu" / "ineI" / "ntel". let is_intel = cpuid0.ebx == 0x756e_6547 && cpuid0.edx == 0x4965_6e69 && cpuid0.ecx == 0x6c65_746e; // "AuthenticAMD" has ebx = 0x68747541 ("Auth") let is_amd = cpuid0.ebx == 0x6874_7541; - // CPUID leaf 1: processor info and feature bits - let cpuid1 = cpuid_leaf(1); - // Extract extended family (bits 27:20) + base family (bits 11:8) let base_family = (cpuid1.eax >> 8) & 0xF; let ext_family = (cpuid1.eax >> 20) & 0xFF; @@ -195,20 +378,23 @@ fn cpuid_batch_x86_64() -> CpuidBatch { // OSXSAVE (bit 27): OS has set CR4.OSXSAVE and supports XSAVE/XGETBV let osxsave = cpuid1.ecx & (1 << 27) != 0; - // Read XCR0 if OSXSAVE is enabled, otherwise assume no extended state support - let xcr0 = if osxsave { - // SAFETY: XGETBV reads XCR0 safely here because: - // 1. CPUID leaf 1 reported OSXSAVE, so the OS enabled XGETBV. - // 2. Index 0 reads XCR0, the architectural extended-state mask. - // 3. The intrinsic returns register state and does not access Rust memory. - unsafe { _xgetbv(0) } - } else { - 0 - }; - // Determine OS support for AVX and AVX-512 register state - let os_avx = (xcr0 & XCR0_AVX_MASK) == XCR0_AVX_MASK; - let os_avx512 = os_avx && (xcr0 & XCR0_AVX512_MASK) == XCR0_AVX512_MASK; + let os_avx = osxsave && (snapshot.xcr0 & XCR0_AVX_MASK) == XCR0_AVX_MASK; + let os_avx512 = os_avx && (snapshot.xcr0 & XCR0_AVX512_MASK) == XCR0_AVX512_MASK; + let os_amx = + osxsave && (snapshot.xcr0 & XCR0_AMX_MASK) == XCR0_AMX_MASK && snapshot.amx_permission; + let os_apx = osxsave && (snapshot.xcr0 & XCR0_APX_MASK) == XCR0_APX_MASK; + + let has_avx = cpuid1.ecx & (1 << 28) != 0; + let has_avx2 = cpuid7.ebx & (1 << 5) != 0; + let has_avx512f = cpuid7.ebx & (1 << 16) != 0; + let has_avx512bw = cpuid7.ebx & (1 << 30) != 0; + let has_fma = cpuid1.ecx & (1 << 12) != 0; + let has_f16c = cpuid1.ecx & (1 << 29) != 0; + let has_aes = cpuid1.ecx & (1 << 25) != 0; + let has_pclmul = cpuid1.ecx & (1 << 1) != 0; + let rust_avx = os_avx && has_avx; + let rust_avx512 = os_avx512 && rust_avx && has_avx512f && has_fma && has_f16c; // ECX features (leaf 1) - SSE/basic features (no OS gating needed) if cpuid1.ecx & (1 << 0) != 0 { @@ -226,10 +412,10 @@ fn cpuid_batch_x86_64() -> CpuidBatch { if cpuid1.ecx & (1 << 23) != 0 { caps |= x86::POPCNT; } - if cpuid1.ecx & (1 << 25) != 0 { + if has_aes { caps |= x86::AESNI; } - if cpuid1.ecx & (1 << 1) != 0 { + if has_pclmul { caps |= x86::PCLMULQDQ; } if cpuid1.ecx & (1 << 30) != 0 { @@ -237,21 +423,16 @@ fn cpuid_batch_x86_64() -> CpuidBatch { } // AVX-class features (require OS AVX support via XCR0) - if os_avx { - if cpuid1.ecx & (1 << 28) != 0 { - caps |= x86::AVX; - } - if cpuid1.ecx & (1 << 12) != 0 { + if rust_avx { + caps |= x86::AVX; + if has_fma { caps |= x86::FMA; } - if cpuid1.ecx & (1 << 29) != 0 { + if has_f16c { caps |= x86::F16C; } } - // Extended feature flags (leaf 7, subleaf 0) - let cpuid7 = cpuid_leaf_count(7, 0); - // EBX features (leaf 7) - non-AVX features (no OS gating needed) if cpuid7.ebx & (1 << 3) != 0 { caps |= x86::BMI1; @@ -267,15 +448,13 @@ fn cpuid_batch_x86_64() -> CpuidBatch { } // AVX2 (requires OS AVX support for YMM registers) - if os_avx && cpuid7.ebx & (1 << 5) != 0 { + if rust_avx && has_avx2 { caps |= x86::AVX2; } - // AVX-512 features (require OS AVX-512 support for ZMM/opmask registers) - if os_avx512 { - if cpuid7.ebx & (1 << 16) != 0 { - caps |= x86::AVX512F; - } + // Rust AVX-512 target features also imply FMA and F16C. + if rust_avx512 { + caps |= x86::AVX512F; if cpuid7.ebx & (1 << 17) != 0 { caps |= x86::AVX512DQ; } @@ -308,87 +487,79 @@ fn cpuid_batch_x86_64() -> CpuidBatch { if cpuid7.ecx & (1 << 14) != 0 { caps |= x86::AVX512VPOPCNTDQ; } - - // Vector extensions that use 512-bit registers (gate with AVX-512 OS support) - if cpuid7.ecx & (1 << 8) != 0 { - caps |= x86::GFNI; + if cpuid7.edx & (1 << 8) != 0 { + caps |= x86::AVX512VP2INTERSECT; } - if cpuid7.ecx & (1 << 9) != 0 { - caps |= x86::VAES; + if has_avx512bw && cpuid7.edx & (1 << 23) != 0 { + caps |= x86::AVX512FP16; } - if cpuid7.ecx & (1 << 10) != 0 { - caps |= x86::VPCLMULQDQ; + if has_avx512bw && cpuid7_1.eax & (1 << 5) != 0 { + caps |= x86::AVX512BF16; } + + if cpuid7_1.edx & (1 << 19) != 0 { + let avx10_version = cpuid24.ebx & 0xFF; + if avx10_version >= 1 { + caps |= x86::AVX10_1; + } + // Rust's AVX10.2 feature also implies AVX-VNNI, AVX-VNNI-INT8, and + // AVX-VNNI-INT16. Caps does not model that prerequisite set, so runtime + // detection deliberately under-reports AVX10_2. + } + } + + if cpuid7.ecx & (1 << 8) != 0 { + caps |= x86::GFNI; + } + if rust_avx && has_avx2 && has_aes && cpuid7.ecx & (1 << 9) != 0 { + caps |= x86::VAES; + } + if rust_avx && has_pclmul && cpuid7.ecx & (1 << 10) != 0 { + caps |= x86::VPCLMULQDQ; } - // EDX features (leaf 7) - non-AVX features - if cpuid7.edx & (1 << 18) != 0 { + // EBX/ECX/EDX features (leaf 7) + if cpuid7.ebx & (1 << 18) != 0 { caps |= x86::RDSEED; } - if cpuid7.edx & (1 << 24) != 0 { + if cpuid7.ecx & (1 << 27) != 0 { + caps |= x86::MOVDIRI; + } + if cpuid7.ecx & (1 << 28) != 0 { + caps |= x86::MOVDIR64B; + } + if cpuid7.edx & (1 << 14) != 0 { + caps |= x86::SERIALIZE; + } + if os_amx && cpuid7.edx & (1 << 24) != 0 { caps |= x86::AMX_TILE; } - if cpuid7.edx & (1 << 22) != 0 { + if os_amx && cpuid7.edx & (1 << 22) != 0 { caps |= x86::AMX_BF16; } - if cpuid7.edx & (1 << 25) != 0 { + if os_amx && cpuid7.edx & (1 << 25) != 0 { caps |= x86::AMX_INT8; } - // Extended feature flags (leaf 7, subleaf 1) - let cpuid7_1 = cpuid_leaf_count(7, 1); - // EAX features (leaf 7, subleaf 1) - // SHA512 doesn't require AVX-512 (uses XMM registers) + // SHA512 doesn't require AVX-512 (uses XMM registers). if cpuid7_1.eax & (1 << 0) != 0 { caps |= x86::SHA512; } - // AVX-512 extensions (require OS AVX-512 support) - if os_avx512 { - if cpuid7_1.eax & (1 << 4) != 0 { - caps |= x86::AVX512BF16; - } - if cpuid7_1.eax & (1 << 5) != 0 { - caps |= x86::AVX512FP16; - } - } - - // AMX extensions (Granite Rapids and newer) - separate state component - // Note: AMX has its own XCR0 bits (17-18), but for now we don't gate these - // as they're not used for crypto kernels - if cpuid7_1.eax & (1 << 21) != 0 { + // AMX extensions require both architectural tile-state components. + if os_amx && cpuid7_1.eax & (1 << 21) != 0 { caps |= x86::AMX_FP16; } - if cpuid7_1.eax & (1 << 8) != 0 { + if os_amx && cpuid7_1.edx & (1 << 8) != 0 { caps |= x86::AMX_COMPLEX; } - // AVX10 detection via CPUID leaf 0x24 (requires OS AVX-512 support) - // AVX10 is Intel's unified vector ISA that subsumes AVX-512 - if os_avx512 && cpuid0.eax >= 0x24 { - let cpuid24 = cpuid_leaf_count(0x24, 0); - let avx10_version = cpuid24.ebx & 0xFF; - if avx10_version >= 1 { - caps |= x86::AVX10_1; - } - if avx10_version >= 2 { - caps |= x86::AVX10_2; - } - } - - // APX detection via CPUID leaf 0x29 - // APX doubles GPRs from 16 to 32 (R16-R31) on Granite Rapids+ - if cpuid0.eax >= 0x29 { - let cpuid29 = cpuid_leaf_count(0x29, 0); - // APX_NCI_NDD_NF is bit 0 of EBX - if cpuid29.ebx & 1 != 0 { - caps |= x86::APX; - } + if os_apx && cpuid7_1.edx & (1 << 21) != 0 { + caps |= x86::APX; } - // Extended CPUID (leaf 0x80000001) for AMD-specific features - let cpuid_ext = cpuid_leaf(0x8000_0001); + // Extended CPUID (leaf 0x80000001) features if cpuid_ext.ecx & (1 << 5) != 0 { caps |= x86::LZCNT; } @@ -416,6 +587,7 @@ fn cpuid_batch_x86_64() -> CpuidBatch { is_amd, family, model, + amx_permission: snapshot.amx_permission, } } @@ -468,10 +640,13 @@ fn runtime_x86_32() -> Caps { /// `RSCRYPTO_FORCE_AVX512=1` enables AVX-512 on hybrid Intel CPUs. #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "std"))] fn hybrid_avx512_override() -> bool { - // Check environment variable for explicit opt-in - std::env::var("RSCRYPTO_FORCE_AVX512") - .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) - .unwrap_or(false) + let value = std::env::var("RSCRYPTO_FORCE_AVX512").ok(); + parse_hybrid_avx512_override(value.as_deref()) +} + +#[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "std"))] +fn parse_hybrid_avx512_override(value: Option<&str>) -> bool { + matches!(value, Some("1")) || value.is_some_and(|value| value.eq_ignore_ascii_case("true")) } #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "std"))] diff --git a/src/platform/detect/compile_time.rs b/src/platform/detect/compile_time.rs index 84faac4c..ce430373 100644 --- a/src/platform/detect/compile_time.rs +++ b/src/platform/detect/compile_time.rs @@ -163,6 +163,9 @@ pub const fn caps_static() -> Caps { if cfg!(target_feature = "sm4") { result = result.union(aarch64::SM3).union(aarch64::SM4); } + if cfg!(target_feature = "sve2-aes") { + result = result.union(aarch64::SVE2_AES).union(aarch64::SVE2_PMULL); + } detect!(result; // Crypto extensions (single-cap) @@ -178,7 +181,6 @@ pub const fn caps_static() -> Caps { // SVE family "sve" => aarch64::SVE, "sve2" => aarch64::SVE2, - "sve2-aes" => aarch64::SVE2_AES, "sve2-sha3" => aarch64::SVE2_SHA3, "sve2-sm4" => aarch64::SVE2_SM4, "sve2-bitperm" => aarch64::SVE2_BITPERM, @@ -298,6 +300,8 @@ pub const fn caps_static() -> Caps { "power8-crypto" => power::POWER8_CRYPTO, "power9-vector" => power::POWER9_VECTOR, "power10-vector" => power::POWER10_VECTOR, + "quadword-atomics" => power::QUADWORD_ATOMICS, + "partword-atomics" => power::PARTWORD_ATOMICS, ); } diff --git a/src/platform/detect/tests.rs b/src/platform/detect/tests.rs index e7fa2722..4659d670 100644 --- a/src/platform/detect/tests.rs +++ b/src/platform/detect/tests.rs @@ -55,6 +55,38 @@ mod tests { assert!(det.caps.has(x86::SSE2)); } + #[test] + #[cfg(target_arch = "x86_64")] + fn x86_64_amx_caps_require_process_permission() { + use crate::platform::caps::x86; + + let caps = x86::SSE2 | X86_ALL_AMX; + assert_eq!(gate_x86_amx_permission(caps, true), caps); + assert_eq!( + gate_x86_amx_permission(caps, false), + x86::SSE2, + "denied process permission must remove every AMX capability" + ); + } + + #[test] + #[cfg(all( + target_arch = "x86_64", + not(feature = "std"), + any(target_os = "linux", target_os = "android"), + target_feature = "amx-tile", + not(miri) + ))] + fn no_std_linux_x86_64_masks_compile_time_amx_without_a_permission_probe() { + use crate::platform::caps::x86; + + assert!(caps_static().has(x86::AMX_TILE)); + assert!( + detect_uncached().caps.intersection(X86_ALL_AMX).is_empty(), + "no_std Linux/Android cannot publish AMX without an xcomp permission probe" + ); + } + #[test] #[cfg(all(target_arch = "aarch64", not(miri)))] fn test_aarch64_baseline() { @@ -109,7 +141,9 @@ mod tests { // is enabled. With `portable-only`, runtime is intentionally `Caps::NONE`, // and `caps_static()` may be non-empty — they're allowed to disagree // because the override is the whole point of the feature. - #[cfg(not(feature = "portable-only"))] + // x86-64 also removes compile-time AMX and hybrid-unsafe AVX-512 + // capabilities when process or CPU-topology safety checks fail. + #[cfg(all(not(feature = "portable-only"), not(target_arch = "x86_64")))] fn test_caps_static_subset_of_runtime() { // Compile-time detected features must be a subset of runtime detected features let static_caps = caps_static(); @@ -215,7 +249,11 @@ mod tests { // Verify the generation is valid assert!(matches!( detected, - AppleSiliconGen::M1 | AppleSiliconGen::M2 | AppleSiliconGen::M3 | AppleSiliconGen::M4 + AppleSiliconGen::M1 + | AppleSiliconGen::M2 + | AppleSiliconGen::M3 + | AppleSiliconGen::M4 + | AppleSiliconGen::M5 )); } } @@ -274,17 +312,14 @@ mod tests { } #[test] - #[allow(unsafe_code)] #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "std"))] - fn test_hybrid_avx512_override_default() { - // Without env var set, override should be false - // Note: We can't easily test with env var set due to test isolation - // but we verify the default behavior - // SAFETY: This test runs in isolation and doesn't rely on this env var being - // present for other threads. The remove_var is unsafe due to potential data - // races with other threads reading env vars, but test isolation mitigates this. - unsafe { std::env::remove_var("RSCRYPTO_FORCE_AVX512") }; - assert!(!hybrid_avx512_override()); + fn test_hybrid_avx512_override_parser() { + for value in [None, Some(""), Some("0"), Some("false"), Some("yes"), Some("2")] { + assert!(!parse_hybrid_avx512_override(value), "accepted {value:?}"); + } + for value in [Some("1"), Some("true"), Some("TRUE"), Some("TrUe")] { + assert!(parse_hybrid_avx512_override(value), "rejected {value:?}"); + } } #[test] @@ -296,6 +331,284 @@ mod tests { assert!(det.caps.count() >= 1); } + #[cfg(all(target_arch = "x86_64", feature = "std"))] + fn cpuid_feature_snapshot() -> CpuidSnapshot { + CpuidSnapshot { + leaf0: CpuidRegisters { + eax: 0x24, + ..CpuidRegisters::default() + }, + leaf1: CpuidRegisters { + ecx: 1 << 27, + ..CpuidRegisters::default() + }, + leaf7_0: CpuidRegisters { + eax: 1, + ..CpuidRegisters::default() + }, + extended_leaf0: CpuidRegisters { + eax: 0x8000_0001, + ..CpuidRegisters::default() + }, + xcr0: 0x6 | 0xe0 | (1 << 17) | (1 << 18) | (1 << 19), + amx_permission: true, + ..CpuidSnapshot::default() + } + } + + #[cfg(all(target_arch = "x86_64", feature = "std"))] + fn enable_avx(snapshot: &mut CpuidSnapshot) { + snapshot.leaf1.ecx |= 1 << 28; + } + + #[cfg(all(target_arch = "x86_64", feature = "std"))] + fn enable_avx512(snapshot: &mut CpuidSnapshot) { + enable_avx(snapshot); + snapshot.leaf1.ecx |= (1 << 12) | (1 << 29); + snapshot.leaf7_0.ebx |= 1 << 16; + } + + #[cfg(all(target_arch = "x86_64", feature = "std"))] + fn avx_caps() -> Caps { + use crate::platform::caps::x86; + + x86::AVX + } + + #[cfg(all(target_arch = "x86_64", feature = "std"))] + fn avx512_caps() -> Caps { + use crate::platform::caps::x86; + + x86::AVX | x86::FMA | x86::F16C | x86::AVX512F + } + + #[test] + #[cfg(all(target_arch = "x86_64", feature = "std"))] + fn x86_cpuid_feature_bits_decode_from_their_architectural_registers() { + use crate::platform::caps::x86; + + fn assert_feature(expected: Caps, configure: impl FnOnce(&mut CpuidSnapshot)) { + let mut snapshot = cpuid_feature_snapshot(); + configure(&mut snapshot); + assert_eq!(decode_cpuid_x86_64(snapshot).caps, expected); + } + + macro_rules! feature_case { + ($expected:expr, $leaf:ident, $register:ident, $bit:expr) => { + assert_feature($expected, |snapshot| snapshot.$leaf.$register |= 1 << $bit); + }; + } + + feature_case!(x86::SSE3, leaf1, ecx, 0); + feature_case!(x86::PCLMULQDQ, leaf1, ecx, 1); + feature_case!(x86::SSSE3, leaf1, ecx, 9); + feature_case!(x86::SSE41, leaf1, ecx, 19); + feature_case!(x86::SSE42, leaf1, ecx, 20); + feature_case!(x86::POPCNT, leaf1, ecx, 23); + feature_case!(x86::AESNI, leaf1, ecx, 25); + feature_case!(x86::RDRAND, leaf1, ecx, 30); + + feature_case!(x86::BMI1, leaf7_0, ebx, 3); + feature_case!(x86::BMI2, leaf7_0, ebx, 8); + feature_case!(x86::RDSEED, leaf7_0, ebx, 18); + feature_case!(x86::ADX, leaf7_0, ebx, 19); + feature_case!(x86::SHA, leaf7_0, ebx, 29); + + feature_case!(x86::GFNI, leaf7_0, ecx, 8); + feature_case!(x86::AMX_BF16, leaf7_0, edx, 22); + feature_case!(x86::AMX_TILE, leaf7_0, edx, 24); + feature_case!(x86::AMX_INT8, leaf7_0, edx, 25); + feature_case!(x86::SHA512, leaf7_1, eax, 0); + feature_case!(x86::AMX_FP16, leaf7_1, eax, 21); + feature_case!(x86::AMX_COMPLEX, leaf7_1, edx, 8); + feature_case!(x86::MOVDIRI, leaf7_0, ecx, 27); + feature_case!(x86::MOVDIR64B, leaf7_0, ecx, 28); + feature_case!(x86::SERIALIZE, leaf7_0, edx, 14); + + feature_case!(x86::LZCNT, extended_leaf1, ecx, 5); + feature_case!(x86::SSE4A, extended_leaf1, ecx, 6); + + assert_feature(avx_caps(), enable_avx); + assert_feature(x86::AVX | x86::FMA, |snapshot| { + enable_avx(snapshot); + snapshot.leaf1.ecx |= 1 << 12; + }); + assert_feature(x86::AVX | x86::F16C, |snapshot| { + enable_avx(snapshot); + snapshot.leaf1.ecx |= 1 << 29; + }); + assert_feature(x86::AVX | x86::AVX2, |snapshot| { + enable_avx(snapshot); + snapshot.leaf7_0.ebx |= 1 << 5; + }); + + let avx512_cases = [ + (x86::AVX512DQ, "ebx", 17), + (x86::AVX512IFMA, "ebx", 21), + (x86::AVX512CD, "ebx", 28), + (x86::AVX512BW, "ebx", 30), + (x86::AVX512VL, "ebx", 31), + (x86::AVX512VBMI, "ecx", 1), + (x86::AVX512VBMI2, "ecx", 6), + (x86::AVX512VNNI, "ecx", 11), + (x86::AVX512BITALG, "ecx", 12), + (x86::AVX512VPOPCNTDQ, "ecx", 14), + (x86::AVX512VP2INTERSECT, "edx", 8), + ]; + for (feature, register, bit) in avx512_cases { + assert_feature(avx512_caps() | feature, |snapshot| { + enable_avx512(snapshot); + match register { + "ebx" => snapshot.leaf7_0.ebx |= 1 << bit, + "ecx" => snapshot.leaf7_0.ecx |= 1 << bit, + "edx" => snapshot.leaf7_0.edx |= 1 << bit, + _ => unreachable!(), + } + }); + } + assert_feature(avx512_caps(), enable_avx512); + + let avx512_bw_caps = avx512_caps() | x86::AVX512BW; + assert_feature(avx512_bw_caps | x86::AVX512FP16, |snapshot| { + enable_avx512(snapshot); + snapshot.leaf7_0.ebx |= 1 << 30; + snapshot.leaf7_0.edx |= 1 << 23; + }); + assert_feature(avx512_bw_caps | x86::AVX512BF16, |snapshot| { + enable_avx512(snapshot); + snapshot.leaf7_0.ebx |= 1 << 30; + snapshot.leaf7_1.eax |= 1 << 5; + }); + + assert_feature(x86::AVX | x86::AVX2 | x86::AESNI | x86::VAES, |snapshot| { + enable_avx(snapshot); + snapshot.leaf1.ecx |= 1 << 25; + snapshot.leaf7_0.ebx |= 1 << 5; + snapshot.leaf7_0.ecx |= 1 << 9; + }); + assert_feature(x86::AVX | x86::PCLMULQDQ | x86::VPCLMULQDQ, |snapshot| { + enable_avx(snapshot); + snapshot.leaf1.ecx |= 1 << 1; + snapshot.leaf7_0.ecx |= 1 << 10; + }); + assert_feature(x86::APX, |snapshot| snapshot.leaf7_1.edx |= 1 << 21); + assert_feature(avx512_caps() | x86::AVX10_1, |snapshot| { + enable_avx512(snapshot); + snapshot.leaf7_1.edx |= 1 << 19; + snapshot.leaf24_0.ebx = 1; + }); + assert_feature(avx512_caps() | x86::AVX10_1, |snapshot| { + enable_avx512(snapshot); + snapshot.leaf7_1.edx |= 1 << 19; + snapshot.leaf24_0.ebx = 2; + }); + } + + #[test] + #[cfg(all(target_arch = "x86_64", feature = "std"))] + fn x86_cpuid_decoder_rejects_unsupported_leaves_and_missing_os_state() { + use crate::platform::caps::x86; + + let mut snapshot = cpuid_feature_snapshot(); + snapshot.leaf0.eax = 6; + snapshot.leaf7_0 = CpuidRegisters { + eax: u32::MAX, + ebx: u32::MAX, + ecx: u32::MAX, + edx: u32::MAX, + }; + snapshot.leaf7_1 = snapshot.leaf7_0; + snapshot.leaf24_0 = snapshot.leaf7_0; + assert!(decode_cpuid_x86_64(snapshot).caps.is_empty()); + + let mut snapshot = cpuid_feature_snapshot(); + snapshot.extended_leaf0.eax = 0x8000_0000; + snapshot.extended_leaf1.ecx = (1 << 5) | (1 << 6); + assert!( + decode_cpuid_x86_64(snapshot) + .caps + .intersection(x86::LZCNT.union(x86::SSE4A)) + .is_empty() + ); + + let mut snapshot = cpuid_feature_snapshot(); + snapshot.leaf7_0.eax = 0; + snapshot.leaf7_1.eax = (1 << 0) | (1 << 4) | (1 << 5) | (1 << 21); + snapshot.leaf7_1.edx = 1 << 8; + assert!(decode_cpuid_x86_64(snapshot).caps.is_empty()); + + let mut snapshot = cpuid_feature_snapshot(); + snapshot.xcr0 = 0; + snapshot.leaf7_0.edx = (1 << 22) | (1 << 24) | (1 << 25); + snapshot.leaf7_1.eax = 1 << 21; + snapshot.leaf7_1.edx = 1 << 8; + let amx = x86::AMX_TILE | x86::AMX_BF16 | x86::AMX_INT8 | x86::AMX_FP16 | x86::AMX_COMPLEX; + assert!(decode_cpuid_x86_64(snapshot).caps.intersection(amx).is_empty()); + + let mut snapshot = cpuid_feature_snapshot(); + snapshot.amx_permission = false; + snapshot.leaf7_0.edx = (1 << 22) | (1 << 24) | (1 << 25); + snapshot.leaf7_1.eax = 1 << 21; + snapshot.leaf7_1.edx = 1 << 8; + assert!(decode_cpuid_x86_64(snapshot).caps.intersection(amx).is_empty()); + + let mut snapshot = cpuid_feature_snapshot(); + snapshot.leaf7_0.edx = 1 << 18; + snapshot.leaf7_1.eax = 1 << 8; + assert!( + decode_cpuid_x86_64(snapshot) + .caps + .intersection(x86::RDSEED | x86::AMX_COMPLEX) + .is_empty() + ); + + let mut snapshot = cpuid_feature_snapshot(); + snapshot.leaf7_0.ebx = 1 << 16; + assert!(decode_cpuid_x86_64(snapshot).caps.intersection(avx512_caps()).is_empty()); + + let mut snapshot = cpuid_feature_snapshot(); + snapshot.xcr0 &= !(1 << 19); + snapshot.leaf7_1.edx = 1 << 21; + assert!(!decode_cpuid_x86_64(snapshot).caps.has(x86::APX)); + + let mut snapshot = cpuid_feature_snapshot(); + enable_avx512(&mut snapshot); + snapshot.leaf24_0.ebx = 2; + assert!( + decode_cpuid_x86_64(snapshot) + .caps + .intersection(x86::AVX10_1 | x86::AVX10_2) + .is_empty() + ); + + for configure in [ + |snapshot: &mut CpuidSnapshot| snapshot.leaf7_0.ecx |= 1 << 9, + |snapshot: &mut CpuidSnapshot| snapshot.leaf7_0.ecx |= 1 << 10, + |snapshot: &mut CpuidSnapshot| snapshot.leaf7_0.ebx |= 1 << 5, + ] { + let mut snapshot = cpuid_feature_snapshot(); + configure(&mut snapshot); + assert!( + decode_cpuid_x86_64(snapshot) + .caps + .intersection(x86::VAES | x86::VPCLMULQDQ | x86::AVX2) + .is_empty() + ); + } + + let mut snapshot = cpuid_feature_snapshot(); + enable_avx512(&mut snapshot); + snapshot.leaf7_0.ebx |= 1 << 30; + snapshot.leaf7_1.eax |= 1 << 4; + assert!(!decode_cpuid_x86_64(snapshot).caps.has(x86::AVX512BF16)); + + let mut snapshot = cpuid_feature_snapshot(); + enable_avx512(&mut snapshot); + snapshot.leaf7_0.ebx |= 1 << 30; + snapshot.leaf7_1.eax |= 1 << 5; + assert!(!decode_cpuid_x86_64(snapshot).caps.has(x86::AVX512FP16)); + } + #[test] #[cfg(all( target_arch = "aarch64", @@ -368,108 +681,6 @@ mod tests { } } - // Arch Round-Trip Tests - - // Mirror of the arch_to_u8 mapping used in atomic_cache (no_std). - // Note: Arch doesn't have #[repr(u8)], so this is a custom mapping - // where Other=0 (the uninitialized/fallback value). - fn test_arch_to_u8(arch: Arch) -> u8 { - match arch { - Arch::X86_64 => 1, - Arch::X86 => 2, - Arch::Aarch64 => 3, - Arch::Arm => 4, - Arch::Riscv64 => 5, - Arch::Riscv32 => 6, - Arch::Power => 7, - Arch::S390x => 8, - Arch::Wasm32 => 10, - Arch::Wasm64 => 11, - Arch::Other => 0, - } - } - - fn test_arch_from_u8(v: u8) -> Arch { - match v { - 1 => Arch::X86_64, - 2 => Arch::X86, - 3 => Arch::Aarch64, - 4 => Arch::Arm, - 5 => Arch::Riscv64, - 6 => Arch::Riscv32, - 7 => Arch::Power, - 8 => Arch::S390x, - 10 => Arch::Wasm32, - 11 => Arch::Wasm64, - _ => Arch::Other, - } - } - - /// Verify arch_to_u8 and arch_from_u8 are inverses. - #[test] - fn test_arch_round_trip() { - let variants: &[Arch] = &[ - Arch::Other, - Arch::X86_64, - Arch::X86, - Arch::Aarch64, - Arch::Arm, - Arch::Riscv64, - Arch::Riscv32, - Arch::Power, - Arch::S390x, - Arch::Wasm32, - Arch::Wasm64, - ]; - - for &arch in variants { - let encoded = test_arch_to_u8(arch); - let decoded = test_arch_from_u8(encoded); - assert_eq!( - arch, decoded, - "Arch round-trip failed: {:?} -> {} -> {:?}", - arch, encoded, decoded - ); - } - - // Verify out-of-range values map to Other - assert_eq!(test_arch_from_u8(12), Arch::Other); - assert_eq!(test_arch_from_u8(255), Arch::Other); - } - - /// Verify all Arch variants have distinct encoded u8 values. - #[test] - fn test_arch_no_collisions() { - use alloc::collections::BTreeSet; - - let variants: &[Arch] = &[ - Arch::Other, - Arch::X86_64, - Arch::X86, - Arch::Aarch64, - Arch::Arm, - Arch::Riscv64, - Arch::Riscv32, - Arch::Power, - Arch::S390x, - Arch::Wasm32, - Arch::Wasm64, - ]; - - let mut seen = BTreeSet::new(); - for &arch in variants { - let val = test_arch_to_u8(arch); - assert!( - seen.insert(val), - "Arch::{:?} has duplicate encoded u8 value {}", - arch, - val - ); - } - - assert_eq!(seen.len(), 11, "Expected 11 Arch variants with unique encodings"); - } - // Override Mechanism Tests #[test] diff --git a/src/platform/mod.rs b/src/platform/mod.rs index 289ed7da..bdef3c2d 100644 --- a/src/platform/mod.rs +++ b/src/platform/mod.rs @@ -9,11 +9,13 @@ //! let runtime = rscrypto::platform::caps(); //! let compile_time = rscrypto::platform::caps_static(); //! -//! // `caps_static()` reports compile-time facts. `caps()` reports runtime -//! // facts. With the default feature set, `caps()` ⊇ `caps_static()`. The -//! // optional `portable-only` feature collapses `caps()` to `Caps::NONE` -//! // for FIPS / DO-178C deployment modes; that override does not change -//! // `caps_static()`. Both functions return `Caps`, a 256-bit bitset. +//! // `caps_static()` reports compile-time facts. `caps()` reports instructions +//! // legal for this process at runtime. Runtime results normally include the +//! // static set, but process-authorized state such as Linux AMX can remove a +//! // static capability. The optional `portable-only` feature collapses +//! // `caps()` to `Caps::NONE` for FIPS / DO-178C deployment modes; that +//! // override does not change `caps_static()`. Both functions return `Caps`, +//! // a 256-bit bitset. //! let _ = (runtime, compile_time); //! ``` //! @@ -28,11 +30,9 @@ //! a mix of both for their own planners. This module does not own dispatch //! policy. //! -//! # Performance -//! -//! - Compile-time capability query: **0ns** after optimization -//! - Cached runtime capability query: **~3ns** -//! - First runtime detection: **~1μs** (CPUID/sysctl, once per process) +//! Compile-time capability queries are constant-foldable. Runtime detection is +//! cached where the target provides the required synchronization; the uncached +//! cost depends on the architecture and operating-system probe. // Core modules pub mod caps; @@ -59,6 +59,11 @@ pub use detect::Detected; /// /// Results are cached after first call. /// +/// On Linux and Android x86_64, AMX capabilities are reported only when the +/// process already has permission for tile state. Detection does not request +/// permission. Request it before the first cached detection if the process +/// intends to use AMX. +/// /// # Examples /// /// ``` diff --git a/src/traits/checksum.rs b/src/traits/checksum.rs index 68115111..642fcc79 100644 --- a/src/traits/checksum.rs +++ b/src/traits/checksum.rs @@ -2,7 +2,7 @@ //! //! Traits for checksum algorithms like CRC32, CRC64, and non-cryptographic hashes. //! -//! - **Performance**: Zero-cost abstractions, inline-friendly +//! - **Performance**: Inline-friendly state updates //! - **Streaming**: Incremental updates for large data //! - **Parallelism**: Combine operation for parallel chunk processing diff --git a/src/traits/ct.rs b/src/traits/ct.rs index 34e55f81..71bbf665 100644 --- a/src/traits/ct.rs +++ b/src/traits/ct.rs @@ -113,7 +113,10 @@ fn byte_difference(left: &[u8], right: &[u8]) -> u64 { #[inline(always)] #[allow(dead_code)] pub(crate) fn fixed_eq(left: &[u8; N], right: &[u8; N]) -> CtDecision { - CtDecision::from_difference(byte_difference(left, right)) + // SECURITY: Keep the accumulated word opaque before declassification. LLVM can otherwise fold + // equality into target-specific vector reductions; exact binary evidence still owns the + // constant-time claim. + CtDecision::from_difference(core::hint::black_box(byte_difference(left, right))) } /// Compare two byte slices whose lengths are public protocol inputs. diff --git a/testdata/auth/wycheproof/README.md b/testdata/auth/wycheproof/README.md new file mode 100644 index 00000000..4e0b3749 --- /dev/null +++ b/testdata/auth/wycheproof/README.md @@ -0,0 +1,15 @@ +# Authentication Wycheproof vectors + +These twelve JSON files are copied byte-for-byte from +[`C2SP/wycheproof`](https://github.com/C2SP/wycheproof) commit +`b61843a9a5115bb758134b6a1f5d5e502d445342`, directory +`testvectors_v1/`. + +Run `scripts/check/auth-vector-provenance.py` to verify the committed file set +and SHA-256 digests. Pass `--upstream-root PATH` to additionally require an +exact checkout of that commit and compare every local file with its upstream +source bytes. + +The corpus is test evidence, not a generated rscrypto artifact. Updating it +requires a new full upstream commit, reviewed digest changes, and corresponding +test-coverage review. diff --git a/testdata/blake2/README.md b/testdata/blake2/README.md new file mode 100644 index 00000000..0c3063ca --- /dev/null +++ b/testdata/blake2/README.md @@ -0,0 +1,15 @@ +# BLAKE2 vector provenance + +`blake2b.blb` and `blake2s.blb` contain the 512 corresponding cases from +`BLAKE2/BLAKE2` commit `ed1974ea83433eba7b2d95c5dcd9ac33cb847913`, +file `testvectors/blake2-kat.json`. The upstream corpus is CC0-1.0 and has +SHA-256 `5031ac14800798ae15cee79c04d65e326a575f2c968c7e2846a79bd07a1c0e61`. + +The deterministic transform emits each case's `in`, `key`, and `out` hex +fields, in source order, using the blobby 0.3 VLQ format with an empty +deduplication table. After checking out the exact commit, reproduce and verify +both files with: + +```bash +scripts/check/hash-vector-provenance.py --blake2-root /path/to/BLAKE2 +``` diff --git a/testdata/blake3/README.md b/testdata/blake3/README.md new file mode 100644 index 00000000..811d1603 --- /dev/null +++ b/testdata/blake3/README.md @@ -0,0 +1,21 @@ +# BLAKE3 vector provenance + +`test_vectors.json` is copied byte-for-byte from `BLAKE3-team/BLAKE3` commit +`8aa5145039b972ba30e98e788752d37d14568824`, file +`test_vectors/test_vectors.json`. The upstream repository provides CC0-1.0 +and Apache-2.0 license files. + +`test_vectors.blb` emits, for each source case, the UTF-8 key, UTF-8 context, +eight-byte little-endian input length, and the three decoded hexadecimal +outputs. It uses the blobby 0.3 VLQ format with an empty deduplication table. +The committed JSON and binary transform are checked by default: + +```bash +scripts/check/hash-vector-provenance.py +``` + +After checking out the exact upstream commit, also verify the source bytes: + +```bash +scripts/check/hash-vector-provenance.py --blake3-root /path/to/BLAKE3 +``` diff --git a/testdata/sha2/README.md b/testdata/sha2/README.md new file mode 100644 index 00000000..fd88ec9f --- /dev/null +++ b/testdata/sha2/README.md @@ -0,0 +1,15 @@ +# SHA-2 vector provenance + +The five `.blb` files are copied byte-for-byte from +`RustCrypto/hashes` commit `82c36a428f8d6f05f3bfccdedb243e9d1f85359d`, +under `sha2/tests/data/`. The upstream repository licenses them under +`MIT OR Apache-2.0`. + +After checking out that exact commit, verify the source bytes with: + +```bash +scripts/check/hash-vector-provenance.py --sha2-root /path/to/RustCrypto-hashes +``` + +The verifier pins every source and destination SHA-256. No conversion is +performed. diff --git a/testdata/sha3/README.md b/testdata/sha3/README.md new file mode 100644 index 00000000..3ad51bb9 --- /dev/null +++ b/testdata/sha3/README.md @@ -0,0 +1,15 @@ +# SHA-3 vector provenance + +The six `.blb` files are copied byte-for-byte from +`RustCrypto/hashes` commit `1637e892b5658941d04a4d895165b66780c7d7ab`, +under `sha3/tests/data/`. The upstream repository licenses them under +`MIT OR Apache-2.0`. + +After checking out that exact commit, verify the source bytes with: + +```bash +scripts/check/hash-vector-provenance.py --sha3-root /path/to/RustCrypto-hashes +``` + +The verifier pins every source and destination SHA-256. No conversion is +performed. diff --git a/tests/cshake256_differential.rs b/tests/cshake256_differential.rs index 50405eda..02cd04f6 100644 --- a/tests/cshake256_differential.rs +++ b/tests/cshake256_differential.rs @@ -1,27 +1,37 @@ #![cfg(feature = "hashes")] +use cshake::{ + CShake128 as OracleCshake128, CShake256 as OracleCshake256, + digest::{ExtendableOutput as _, Update as _, XofReader as _}, +}; use proptest::prelude::*; use rscrypto::{Cshake128, Cshake256, traits::Xof as _}; fn cshake128_ref(function_name: &[u8], customization: &[u8], data: &[u8], out: &mut [u8]) { - use tiny_keccak::{CShake, Hasher as _}; - - let mut hasher = CShake::v128(function_name, customization); + let mut hasher = OracleCshake128::new_with_function_name(function_name, customization); hasher.update(data); - hasher.finalize(out); + hasher.finalize_xof().read(out); } fn cshake256_ref(function_name: &[u8], customization: &[u8], data: &[u8], out: &mut [u8]) { - use tiny_keccak::{CShake, Hasher as _}; - - let mut hasher = CShake::v256(function_name, customization); + let mut hasher = OracleCshake256::new_with_function_name(function_name, customization); hasher.update(data); - hasher.finalize(out); + hasher.finalize_xof().read(out); +} + +fn decode_hex_64(value: &str) -> [u8; 64] { + assert_eq!(value.len(), 128); + let mut out = [0u8; 64]; + for (index, byte) in out.iter_mut().enumerate() { + let offset = index * 2; + *byte = u8::from_str_radix(&value[offset..offset + 2], 16).unwrap(); + } + out } proptest! { #[test] - fn cshake128_one_shot_matches_tiny_keccak( + fn cshake128_one_shot_matches_rustcrypto( function_name in proptest::collection::vec(any::(), 0..192), customization in proptest::collection::vec(any::(), 0..192), data in proptest::collection::vec(any::(), 0..4096), @@ -37,7 +47,7 @@ proptest! { } #[test] - fn cshake128_streaming_matches_tiny_keccak( + fn cshake128_streaming_matches_rustcrypto( function_name in proptest::collection::vec(any::(), 0..192), customization in proptest::collection::vec(any::(), 0..192), data in proptest::collection::vec(any::(), 0..4096), @@ -61,7 +71,7 @@ proptest! { } #[test] - fn cshake256_one_shot_matches_tiny_keccak( + fn cshake256_one_shot_matches_rustcrypto( function_name in proptest::collection::vec(any::(), 0..192), customization in proptest::collection::vec(any::(), 0..192), data in proptest::collection::vec(any::(), 0..4096), @@ -77,7 +87,7 @@ proptest! { } #[test] - fn cshake256_streaming_matches_tiny_keccak( + fn cshake256_streaming_matches_rustcrypto( function_name in proptest::collection::vec(any::(), 0..192), customization in proptest::collection::vec(any::(), 0..192), data in proptest::collection::vec(any::(), 0..4096), @@ -102,39 +112,46 @@ proptest! { } #[test] -fn cshake128_exact_rate_prefix_matches_tiny_keccak() { - use tiny_keccak::{Hasher as _, Xof as _}; - - let function_name = [0xff; 161]; - let mut expected = [0u8; 255]; - let mut oracle = tiny_keccak::CShake::v128(&function_name, b""); - oracle.update(b""); - oracle.squeeze(&mut expected[..167]); - oracle.squeeze(&mut expected[167..]); - - let mut actual = [0u8; 255]; - let mut reader = Cshake128::new(&function_name, b"").finalize_xof(); - reader.squeeze(&mut actual[..167]); - reader.squeeze(&mut actual[167..]); - - assert_eq!(actual, expected); -} +fn cshake_bytepad_rate_boundaries_match_go_crypto_sha3() { + // Generated independently with Go 1.26.5's standard-library + // crypto/sha3.NewCSHAKE128 and NewCSHAKE256. + let cases128 = [ + ( + 160, + "99aba19b4ac53e5df9aa569831ee87fc3cb063731b03abd73f3ac54d6b93a437dd17fab8c0961be98b036179f212bba251f8ec0000a3ed1fb5121e9ac1c564bc", + ), + ( + 161, + "957923a2379d6fde510a97f8d53af1131f543e8080fcb1a5db4acbb15bea189f6f81a0986827ce523673d9947dec1c0b99edaa443cc492cfbabeccfd951c1299", + ), + ( + 162, + "092ddeceaa1c4d98c91bbae1de6c971988f000c257c4c4ed89792698be2145347b9bc3589c8317c9654207f3b8fb543630c91ce227312a48429841161dcac583", + ), + ]; + for (name_len, expected) in cases128 { + let mut actual = [0u8; 64]; + Cshake128::hash_into(&vec![0xff; name_len], b"", b"", &mut actual); + assert_eq!(actual, decode_hex_64(expected), "cSHAKE128 N={name_len}"); + } -#[test] -fn cshake256_exact_rate_prefix_matches_tiny_keccak() { - use tiny_keccak::{Hasher as _, Xof as _}; - - let function_name = [0xff; 129]; - let mut expected = [0u8; 255]; - let mut oracle = tiny_keccak::CShake::v256(&function_name, b""); - oracle.update(b""); - oracle.squeeze(&mut expected[..137]); - oracle.squeeze(&mut expected[137..]); - - let mut actual = [0u8; 255]; - let mut reader = Cshake256::new(&function_name, b"").finalize_xof(); - reader.squeeze(&mut actual[..137]); - reader.squeeze(&mut actual[137..]); - - assert_eq!(actual, expected); + let cases256 = [ + ( + 128, + "a85b94a121902b2e16fad687bbbc27698f6cb9517f49567d0b925abd93f794408ad99a30c61e626cbd216525505aac7c3cbcbd9fe02ad0381eb2bccf60e8e989", + ), + ( + 129, + "495d281b373f64e33ea5e96efc3e13a6da5897397e02cc9f6e5c9f9e03312ff116185a092f41ed5f9bbaf18db0d31d6135cad43308400a07f70d446a630263fe", + ), + ( + 130, + "39887e69a44f8e722f6597285f78841ae51ec3c65447f8f7471be1a257003fdf3d550afa356e323c561fb7eb8e9ff25720cf99458734d8bb245376d79f9533a6", + ), + ]; + for (name_len, expected) in cases256 { + let mut actual = [0u8; 64]; + Cshake256::hash_into(&vec![0xff; name_len], b"", b"", &mut actual); + assert_eq!(actual, decode_hex_64(expected), "cSHAKE256 N={name_len}"); + } } diff --git a/tests/kmac128_differential.rs b/tests/kmac128_differential.rs index 2fdced2d..4a6e7750 100644 --- a/tests/kmac128_differential.rs +++ b/tests/kmac128_differential.rs @@ -11,6 +11,26 @@ fn kmac128_ref(key: &[u8], customization: &[u8], data: &[u8], out: &mut [u8]) { kmac.finalize(out); } +fn encoded_string_len(len: usize) -> usize { + let bits = len * 8; + let width = ((usize::BITS - bits.leading_zeros()) as usize).div_ceil(8).max(1); + 1 + width + len +} + +fn bytepad_is_aligned(rate: usize, segments: &[usize]) -> bool { + (2 + segments.iter().map(|&len| encoded_string_len(len)).sum::()).is_multiple_of(rate) +} + +fn decode_hex_32(value: &str) -> [u8; 32] { + assert_eq!(value.len(), 64); + let mut out = [0u8; 32]; + for (index, byte) in out.iter_mut().enumerate() { + let offset = index * 2; + *byte = u8::from_str_radix(&value[offset..offset + 2], 16).unwrap(); + } + out +} + proptest! { #[test] fn kmac128_matches_tiny_keccak( @@ -19,6 +39,8 @@ proptest! { data in proptest::collection::vec(any::(), 0..4096), out_len in 0usize..256, ) { + prop_assume!(!bytepad_is_aligned(168, &[4, customization.len()])); + prop_assume!(!bytepad_is_aligned(168, &[key.len()])); let mut expected = vec![0u8; out_len]; kmac128_ref(&key, &customization, &data, &mut expected); @@ -29,7 +51,14 @@ proptest! { if expected.is_empty() { prop_assert!(Kmac128::verify_tag(&key, &customization, &data, &expected).is_err()); } else { - prop_assert_eq!(Kmac128::verify_tag(&key, &customization, &data, &expected), Ok(())); + prop_assert_eq!( + Kmac128::verify_tag(&key, &customization, &data, &expected).is_ok(), + expected.len() >= Kmac128::MIN_AUTH_TAG_SIZE + ); + prop_assert_eq!( + Kmac128::verify_tag_primitive(&key, &customization, &data, &expected), + Ok(()) + ); } } @@ -40,6 +69,8 @@ proptest! { data in proptest::collection::vec(any::(), 0..4096), out_len in 0usize..256, ) { + prop_assume!(!bytepad_is_aligned(168, &[4, customization.len()])); + prop_assume!(!bytepad_is_aligned(168, &[key.len()])); let mut expected = vec![0u8; out_len]; kmac128_ref(&key, &customization, &data, &mut expected); @@ -57,3 +88,13 @@ proptest! { prop_assert_eq!(actual, expected); } } + +#[test] +fn kmac128_exact_rate_key_bytepad_matches_openssl() { + // OpenSSL 3.6.3 KMAC-128, empty customization/message, 32-byte output. + let key = [0x42; 163]; + let expected = decode_hex_32("6c9c526b592fdabce43190f544cf5ae6671d223d268eb5a206283df289346fc8"); + let mut actual = [0u8; 32]; + Kmac128::mac_into(&key, b"", b"", &mut actual); + assert_eq!(actual, expected); +} diff --git a/tests/kmac256_differential.rs b/tests/kmac256_differential.rs index 53739f0b..707d14b3 100644 --- a/tests/kmac256_differential.rs +++ b/tests/kmac256_differential.rs @@ -11,6 +11,26 @@ fn kmac256_ref(key: &[u8], customization: &[u8], data: &[u8], out: &mut [u8]) { kmac.finalize(out); } +fn encoded_string_len(len: usize) -> usize { + let bits = len * 8; + let width = ((usize::BITS - bits.leading_zeros()) as usize).div_ceil(8).max(1); + 1 + width + len +} + +fn bytepad_is_aligned(rate: usize, segments: &[usize]) -> bool { + (2 + segments.iter().map(|&len| encoded_string_len(len)).sum::()).is_multiple_of(rate) +} + +fn decode_hex_64(value: &str) -> [u8; 64] { + assert_eq!(value.len(), 128); + let mut out = [0u8; 64]; + for (index, byte) in out.iter_mut().enumerate() { + let offset = index * 2; + *byte = u8::from_str_radix(&value[offset..offset + 2], 16).unwrap(); + } + out +} + proptest! { #[test] fn kmac256_matches_tiny_keccak( @@ -19,6 +39,8 @@ proptest! { data in proptest::collection::vec(any::(), 0..4096), out_len in 0usize..256, ) { + prop_assume!(!bytepad_is_aligned(136, &[4, customization.len()])); + prop_assume!(!bytepad_is_aligned(136, &[key.len()])); let mut expected = vec![0u8; out_len]; kmac256_ref(&key, &customization, &data, &mut expected); @@ -29,7 +51,14 @@ proptest! { if expected.is_empty() { prop_assert!(Kmac256::verify_tag(&key, &customization, &data, &expected).is_err()); } else { - prop_assert_eq!(Kmac256::verify_tag(&key, &customization, &data, &expected), Ok(())); + prop_assert_eq!( + Kmac256::verify_tag(&key, &customization, &data, &expected).is_ok(), + expected.len() >= Kmac256::MIN_AUTH_TAG_SIZE + ); + prop_assert_eq!( + Kmac256::verify_tag_primitive(&key, &customization, &data, &expected), + Ok(()) + ); } } @@ -40,6 +69,8 @@ proptest! { data in proptest::collection::vec(any::(), 0..4096), out_len in 0usize..256, ) { + prop_assume!(!bytepad_is_aligned(136, &[4, customization.len()])); + prop_assume!(!bytepad_is_aligned(136, &[key.len()])); let mut expected = vec![0u8; out_len]; kmac256_ref(&key, &customization, &data, &mut expected); @@ -59,17 +90,13 @@ proptest! { } #[test] -fn kmac256_exact_rate_key_bytepad_matches_tiny_keccak() { - use tiny_keccak::Hasher as _; - +fn kmac256_exact_rate_key_bytepad_matches_openssl() { + // OpenSSL 3.6.3 KMAC-256, empty customization/message, 64-byte output. let key = [0x42; 131]; - let mut expected = [0u8; 64]; - let mut oracle = tiny_keccak::Kmac::v256(&key, b""); - oracle.update(b""); - oracle.finalize(&mut expected); - + let expected = decode_hex_64( + "875ea09c011f7ab1f6238aeac8bc0f88951a567be7447cd23a6c6187a086c94a64202d2c1f46ab1ddfdff61d173eba49fcf3039a70d088c908b46f3c3693a6d9", + ); let mut actual = [0u8; 64]; Kmac256::mac_into(&key, b"", b"", &mut actual); - assert_eq!(actual, expected); } diff --git a/tests/kmac_wycheproof.rs b/tests/kmac_wycheproof.rs index 79fe4fd8..2b43d65b 100644 --- a/tests/kmac_wycheproof.rs +++ b/tests/kmac_wycheproof.rs @@ -49,14 +49,14 @@ fn kmac256_no_customization_wycheproof_vectors() { Kmac256::mac_into(&key, b"", &msg, &mut actual); assert_eq!(actual, tag, "KMAC256 tcId {tc_id} tag mismatch"); assert!( - Kmac256::verify_tag(&key, b"", &msg, &tag).is_ok(), + Kmac256::verify_tag_primitive(&key, b"", &msg, &tag).is_ok(), "KMAC256 tcId {tc_id} verify failed" ); } "invalid" => { counts.invalid += 1; assert!( - Kmac256::verify_tag(&key, b"", &msg, &tag).is_err(), + Kmac256::verify_tag_primitive(&key, b"", &msg, &tag).is_err(), "KMAC256 tcId {tc_id} accepted an invalid tag" ); } diff --git a/tests/pbkdf2_kat_vectors.rs b/tests/pbkdf2_kat_vectors.rs index 0fe384c5..cbc4ab1d 100644 --- a/tests/pbkdf2_kat_vectors.rs +++ b/tests/pbkdf2_kat_vectors.rs @@ -1,17 +1,18 @@ //! PBKDF2-HMAC-SHA256 / SHA-512 known-answer vectors. //! //! RFC 6070 only covers PBKDF2-HMAC-SHA1. rscrypto ships SHA-256 and SHA-512 -//! variants, so this file pins them against widely-published test vectors -//! that predate any one vendor: +//! variants, so this file pins them against the vectors in +//! `ring 0.16.20/tests/pbkdf2_tests.txt`: //! -//! - SHA-256: IETF draft-josefsson-scrypt-kdf-01 §10 / Stack Overflow's canonical -//! PBKDF2-HMAC-SHA256 vectors (also reproduced by RustCrypto `pbkdf2` crate tests). -//! - SHA-512: RFC 7914 companion values (and the RustCrypto test fixtures). +//! - SHA-256 source cited there: +//! - SHA-512 source cited there: //! -//! The differential suite in `tests/pbkdf2_differential.rs` already pins -//! rscrypto against the RustCrypto `pbkdf2` crate across many parameter -//! triples; this file adds a frozen, reviewer-friendly KAT surface that -//! does not depend on any external oracle. +//! RFC 7914 specifies scrypt; it is not the provenance of the SHA-512 values. +//! The shared cases are also present in RustCrypto `pbkdf2 0.13.0`, and +//! `tests/pbkdf2_differential.rs` compares rscrypto against that independent +//! implementation across generated parameter triples. This file keeps a +//! frozen, reviewer-friendly KAT surface that does not invoke an external +//! oracle at test time. #![cfg(feature = "pbkdf2")] use rscrypto::{Pbkdf2Sha256, Pbkdf2Sha512}; diff --git a/tests/platform_amx_permission.rs b/tests/platform_amx_permission.rs new file mode 100644 index 00000000..d8837a15 --- /dev/null +++ b/tests/platform_amx_permission.rs @@ -0,0 +1,187 @@ +#![cfg(all( + feature = "std", + not(feature = "portable-only"), + target_arch = "x86_64", + target_os = "linux", + not(miri) +))] +#![allow(unsafe_code)] + +use rscrypto::platform::{self, caps::x86}; + +const CHILD_MODE: &str = "RSCRYPTO_PLATFORM_AMX_CHILD"; +const REQUIRE_AMX: &str = "RSCRYPTO_REQUIRE_AMX"; +const CACHE_TRANSITION: &str = "cache-transition"; +const REQUEST_BEFORE_CACHE: &str = "request-before-cache"; +const ARCH_GET_XCOMP_PERM: usize = 0x1022; +const ARCH_REQ_XCOMP_PERM: usize = 0x1023; +const XFEATURE_XTILEDATA: usize = 18; +const XCOMP_TILE_MASK: u64 = (1 << 17) | (1 << 18); + +#[allow(unused_unsafe)] +fn cpu_supports_amx_tile() -> bool { + // MSRV: CPUID is unsafe on Rust 1.91 but safe on the pinned nightly. + // SAFETY: CPUID is a non-privileged x86-64 identification instruction. + let leaf0 = unsafe { core::arch::x86_64::__cpuid(0) }; + if leaf0.eax < 7 { + return false; + } + // SAFETY: CPUID leaf 7, subleaf 0 is valid because leaf 0 reports support + // for leaf 7; the intrinsic only returns register values. + let leaf7 = unsafe { core::arch::x86_64::__cpuid_count(7, 0) }; + leaf7.edx & (1 << 24) != 0 +} + +fn xcomp_permissions() -> Option { + const SYS_ARCH_PRCTL: isize = 158; + + let mut permissions = 0u64; + let mut result = SYS_ARCH_PRCTL; + + // SAFETY: Linux x86-64 syscall ABI invocation because: + // 1. syscall 158 is arch_prctl on this test's cfg-constrained target; + // 2. ARCH_GET_XCOMP_PERM writes one u64 through the valid exclusive pointer in RSI and does not + // retain it; and + // 3. syscall clobbers RAX, RCX, and R11, all of which are declared. + unsafe { + core::arch::asm!( + "syscall", + inlateout("rax") result, + in("rdi") ARCH_GET_XCOMP_PERM, + in("rsi") &mut permissions, + lateout("rcx") _, + lateout("r11") _, + options(nostack), + ); + } + + (result == 0).then_some(permissions) +} + +fn request_tile_data_permission() -> bool { + const SYS_ARCH_PRCTL: isize = 158; + + let mut result = SYS_ARCH_PRCTL; + + // SAFETY: Linux x86-64 syscall ABI invocation because: + // 1. syscall 158 is arch_prctl on this test's cfg-constrained target; + // 2. ARCH_REQ_XCOMP_PERM consumes the immediate XFEATURE_XTILEDATA number in RSI and does not + // dereference it; and + // 3. syscall clobbers RAX, RCX, and R11, all of which are declared. + unsafe { + core::arch::asm!( + "syscall", + inlateout("rax") result, + in("rdi") ARCH_REQ_XCOMP_PERM, + in("rsi") XFEATURE_XTILEDATA, + lateout("rcx") _, + lateout("r11") _, + options(nostack), + ); + } + + result == 0 +} + +fn require_amx() -> bool { + std::env::var_os(REQUIRE_AMX).is_some() +} + +fn skip_or_fail(reason: &str) { + if require_amx() { + panic!("{reason}"); + } + eprintln!("skipping AMX permission transition: {reason}"); +} + +fn cache_transition_child() { + if !cpu_supports_amx_tile() { + skip_or_fail("CPU does not report AMX-TILE"); + return; + } + + let Some(before_permissions) = xcomp_permissions() else { + skip_or_fail("ARCH_GET_XCOMP_PERM is unavailable"); + return; + }; + if require_amx() { + assert_eq!( + before_permissions & XCOMP_TILE_MASK, + 1 << 17, + "fresh process must begin with tile configuration but not tile-data permission" + ); + } + + let before = platform::caps(); + assert_eq!( + before.has(x86::AMX_TILE), + before_permissions & XCOMP_TILE_MASK == XCOMP_TILE_MASK, + "cached detection must match the process permission visible at initialization" + ); + + if !request_tile_data_permission() { + skip_or_fail("ARCH_REQ_XCOMP_PERM rejected XFEATURE_XTILEDATA"); + return; + } + let after_permissions = xcomp_permissions().expect("permission query must succeed after a successful request"); + assert_eq!( + after_permissions & XCOMP_TILE_MASK, + XCOMP_TILE_MASK, + "successful tile-data request must authorize both AMX state components" + ); + + assert_eq!( + platform::caps(), + before, + "cached detection must not silently change after initialization" + ); + assert!( + platform::expert::detect_uncached().caps.has(x86::AMX_TILE), + "fresh detection after permission must publish AMX-TILE" + ); +} + +fn request_before_cache_child() { + if !cpu_supports_amx_tile() { + skip_or_fail("CPU does not report AMX-TILE"); + return; + } + if !request_tile_data_permission() { + skip_or_fail("ARCH_REQ_XCOMP_PERM rejected XFEATURE_XTILEDATA"); + return; + } + + assert!( + platform::caps().has(x86::AMX_TILE), + "permission requested before cached detection must make AMX-TILE available" + ); +} + +#[test] +fn linux_x86_64_amx_permission_and_cache_are_process_scoped() { + match std::env::var(CHILD_MODE).as_deref() { + Ok(CACHE_TRANSITION) => { + cache_transition_child(); + return; + } + Ok(REQUEST_BEFORE_CACHE) => { + request_before_cache_child(); + return; + } + Ok(other) => panic!("unknown AMX child mode: {other}"), + Err(std::env::VarError::NotPresent) => {} + Err(error) => panic!("invalid AMX child mode: {error}"), + } + + let executable = std::env::current_exe().expect("current test executable"); + for mode in [CACHE_TRANSITION, REQUEST_BEFORE_CACHE] { + let status = std::process::Command::new(&executable) + .arg("--exact") + .arg("linux_x86_64_amx_permission_and_cache_are_process_scoped") + .arg("--nocapture") + .env(CHILD_MODE, mode) + .status() + .expect("spawn isolated AMX detector process"); + assert!(status.success(), "AMX detector child failed in mode {mode}"); + } +} diff --git a/tests/platform_override_race.rs b/tests/platform_override_race.rs index 37f82b2b..b6f78f72 100644 --- a/tests/platform_override_race.rs +++ b/tests/platform_override_race.rs @@ -2,6 +2,8 @@ use rscrypto::platform::{Detected, expert}; +const CHILD_MODE: &str = "RSCRYPTO_PLATFORM_OVERRIDE_RACE_CHILD"; + #[test] fn concurrent_override_writers_are_serialized() { std::thread::scope(|scope| { @@ -17,3 +19,50 @@ fn concurrent_override_writers_are_serialized() { expert::try_set_override(None).unwrap(); } + +#[test] +#[cfg(not(miri))] +fn concurrent_detection_and_override_child() { + if std::env::var_os(CHILD_MODE).is_none() { + return; + } + + let barrier = std::sync::Barrier::new(3); + std::thread::scope(|scope| { + let setter = scope.spawn(|| { + barrier.wait(); + expert::try_set_override(Some(Detected::portable())) + }); + let detector = scope.spawn(|| { + barrier.wait(); + rscrypto::platform::get() + }); + + barrier.wait(); + let setter_result = setter.join().unwrap(); + let detected = detector.join().unwrap(); + + match setter_result { + Ok(()) => assert_eq!(detected, Detected::portable()), + Err(expert::OverrideError::AlreadyInitialized) => {} + Err(error) => panic!("unexpected override result: {error:?}"), + } + }); +} + +#[test] +#[cfg(not(miri))] +fn concurrent_detection_and_override_are_linearizable() { + let executable = std::env::current_exe().unwrap(); + + for _ in 0..32 { + let status = std::process::Command::new(&executable) + .arg("--exact") + .arg("concurrent_detection_and_override_child") + .arg("--quiet") + .env(CHILD_MODE, "1") + .status() + .unwrap(); + assert!(status.success(), "race child failed with {status}"); + } +} diff --git a/tests/rsa_nist_cavp.rs b/tests/rsa_nist_cavp.rs index 946dfffe..b5ed390b 100644 --- a/tests/rsa_nist_cavp.rs +++ b/tests/rsa_nist_cavp.rs @@ -321,6 +321,80 @@ fn nist_cavp_sha2_siggen_private_operations_match_expected_signatures() { assert_eq!(coverage, expected_siggen_coverage()); } +#[test] +fn nist_cavp_same_width_public_scratch_rebinds_between_keys() { + let suite: Value = serde_json::from_str(CAVP_SIGGEN_186_3_PRIVATE).expect("CAVP SigGen JSON must parse"); + let tests = cavp_tests(&suite); + let policy = RsaPublicKeyPolicy::legacy_verification().allow_legacy_odd_exponents(); + let key_a = private_key_from_cavp_siggen(&tests[0], &policy); + let key_b = private_key_from_cavp_siggen(&tests[6], &policy); + assert_eq!(key_a.public_key().modulus().len(), key_b.public_key().modulus().len()); + assert_ne!(key_a.public_key().modulus(), key_b.public_key().modulus()); + + let representative_b = hex_to_vec(field(&tests[6], "sig")); + let mut expected_b = vec![0u8; key_b.public_key().modulus().len()]; + key_b + .public_key() + .public_operation(&representative_b, &mut expected_b) + .expect("key-B public operation must succeed"); + + let mut scratch = key_a.public_key().public_scratch(); + let mut actual_b = vec![0u8; expected_b.len()]; + key_b + .public_key() + .public_operation_with_scratch(&representative_b, &mut actual_b, &mut scratch) + .expect("same-width scratch must rebind from key A to key B"); + assert_eq!(actual_b, expected_b); + + let mut representative_a = vec![0u8; key_a.public_key().modulus().len()]; + *representative_a.last_mut().expect("non-empty RSA modulus") = 2; + let mut expected_a = vec![0u8; representative_a.len()]; + key_a + .public_key() + .public_operation(&representative_a, &mut expected_a) + .expect("key-A public operation must succeed"); + + let mut actual_a = vec![0u8; expected_a.len()]; + key_a + .public_key() + .public_operation_with_scratch(&representative_a, &mut actual_a, &mut scratch) + .expect("same-width scratch must rebind from key B back to key A"); + assert_eq!(actual_a, expected_a); +} + +#[test] +fn nist_cavp_same_width_private_scratch_rebinds_between_keys() { + let suite: Value = serde_json::from_str(CAVP_SIGGEN_186_3_PRIVATE).expect("CAVP SigGen JSON must parse"); + let tests = cavp_tests(&suite); + let policy = RsaPublicKeyPolicy::legacy_verification().allow_legacy_odd_exponents(); + let key_a = private_key_from_cavp_siggen(&tests[0], &policy); + let test_b = &tests[6]; + let key_b = private_key_from_cavp_siggen(test_b, &policy); + assert_eq!(key_a.public_key().modulus().len(), key_b.public_key().modulus().len()); + assert_ne!(key_a.public_key().modulus(), key_b.public_key().modulus()); + + let message = hex_to_vec(field(test_b, "msg")); + let salt = hex_to_vec(field(test_b, "salt")); + let expected_signature = hex_to_vec(field(test_b, "sig")); + let blinding_factor = fixed_width_one(key_b.public_key().modulus().len()); + let blinding_factor_inverse = fixed_width_one(key_b.public_key().modulus().len()); + let mut signature = vec![0u8; expected_signature.len()]; + let mut scratch = key_a.private_scratch(); + + key_b + .sign_pss_with_salt_and_blinding_factor_and_scratch( + pss_profile(field(test_b, "sha")), + &message, + &salt, + &blinding_factor, + &blinding_factor_inverse, + &mut signature, + &mut scratch, + ) + .expect("same-width private scratch must rebind from key A to key B"); + assert_eq!(signature, expected_signature); +} + #[cfg(feature = "getrandom")] #[test] fn nist_cavp_sha2_siggen_profile_signing_matches_expected_results() { diff --git a/tests/rsa_wycheproof.rs b/tests/rsa_wycheproof.rs index 91d906ce..5daded75 100644 --- a/tests/rsa_wycheproof.rs +++ b/tests/rsa_wycheproof.rs @@ -1,5 +1,7 @@ #![cfg(feature = "rsa")] +use core::ops::Range; + use rscrypto::{RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKey, RsaPublicKeyPolicy}; use serde_json::Value; @@ -43,6 +45,8 @@ const OAEP_4096_SHA512_MGF1SHA1: &str = const RSAES_PKCS1_2048: &str = include_str!("../testdata/rsa/wycheproof/rsa_pkcs1_2048_test.json"); const RSAES_PKCS1_3072: &str = include_str!("../testdata/rsa/wycheproof/rsa_pkcs1_3072_test.json"); const RSAES_PKCS1_4096: &str = include_str!("../testdata/rsa/wycheproof/rsa_pkcs1_4096_test.json"); +const RSAES_PKCS1_GROUP_COUNT: usize = 33; +const RSAES_PKCS1_TEST_COUNT: u64 = 67; #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct ExpectedCounts { @@ -525,18 +529,39 @@ fn assert_oaep_mgf1sha1_vectors_are_rejected( ); } -fn assert_rsaes_pkcs1v15_wycheproof_vectors(json: &str, expected_key_size: u64, expected: ExpectedCounts) { +fn assert_rsaes_pkcs1v15_wycheproof_vectors( + json: &str, + expected_key_size: u64, + group_range: Range, + expected: ExpectedCounts, +) { let suite: Value = serde_json::from_str(json).expect("Wycheproof RSAES-PKCS1-v1_5 JSON must parse"); assert_eq!(suite["algorithm"].as_str(), Some("RSAES-PKCS1-v1_5")); + let all_groups = groups(&suite, "RSAES-PKCS1-v1_5"); + assert_eq!(all_groups.len(), RSAES_PKCS1_GROUP_COUNT); + assert_eq!(suite["numberOfTests"].as_u64(), Some(RSAES_PKCS1_TEST_COUNT)); assert_eq!( - suite["numberOfTests"].as_u64(), - Some(expected.valid.strict_add(expected.invalid) as u64) + all_groups.iter().map(|group| test_cases(group).len()).sum::() as u64, + RSAES_PKCS1_TEST_COUNT + ); + let selected_groups = all_groups + .get(group_range) + .expect("Wycheproof RSAES-PKCS1-v1_5 group range must exist"); + assert_eq!( + selected_groups + .iter() + .map(|group| test_cases(group).len()) + .sum::(), + expected + .valid + .strict_add(expected.acceptable) + .strict_add(expected.invalid) ); let mut valid = 0usize; let mut invalid = 0usize; let mut acceptable = 0usize; - for group in groups(&suite, "RSAES-PKCS1-v1_5") { + for group in selected_groups { assert_eq!(group["type"].as_str(), Some("RsaesPkcs1Decrypt")); assert_eq!(group["keySize"].as_u64(), Some(expected_key_size)); let private_key_der = hex_to_vec(field(group, "privateKeyPkcs8")); @@ -859,32 +884,155 @@ fn wycheproof_pkcs1v15_sha2_sig_gen_vectors_match_expected_signatures() { } #[test] -fn wycheproof_rsaes_pkcs1v15_decrypt_vectors_match_expected_results() { +fn wycheproof_rsaes_pkcs1v15_2048_decrypt_vectors_match_expected_results() { assert_rsaes_pkcs1v15_wycheproof_vectors( RSAES_PKCS1_2048, 2048, + 0..RSAES_PKCS1_GROUP_COUNT, ExpectedCounts { valid: 42, acceptable: 0, invalid: 25, }, ); +} + +#[test] +fn wycheproof_rsaes_pkcs1v15_3072_decrypt_vectors_match_expected_results() { assert_rsaes_pkcs1v15_wycheproof_vectors( RSAES_PKCS1_3072, 3072, + 0..RSAES_PKCS1_GROUP_COUNT, ExpectedCounts { valid: 41, acceptable: 0, invalid: 26, }, ); +} + +#[test] +fn wycheproof_rsaes_pkcs1v15_4096_cases_1_to_35_match_expected_results() { assert_rsaes_pkcs1v15_wycheproof_vectors( RSAES_PKCS1_4096, 4096, + 0..1, ExpectedCounts { - valid: 41, + valid: 10, acceptable: 0, - invalid: 26, + invalid: 25, + }, + ); +} + +#[test] +fn wycheproof_rsaes_pkcs1v15_4096_cases_36_to_39_match_expected_results() { + assert_rsaes_pkcs1v15_wycheproof_vectors( + RSAES_PKCS1_4096, + 4096, + 1..5, + ExpectedCounts { + valid: 4, + acceptable: 0, + invalid: 0, + }, + ); +} + +#[test] +fn wycheproof_rsaes_pkcs1v15_4096_cases_40_to_43_match_expected_results() { + assert_rsaes_pkcs1v15_wycheproof_vectors( + RSAES_PKCS1_4096, + 4096, + 5..9, + ExpectedCounts { + valid: 4, + acceptable: 0, + invalid: 0, + }, + ); +} + +#[test] +fn wycheproof_rsaes_pkcs1v15_4096_cases_44_to_47_match_expected_results() { + assert_rsaes_pkcs1v15_wycheproof_vectors( + RSAES_PKCS1_4096, + 4096, + 9..13, + ExpectedCounts { + valid: 4, + acceptable: 0, + invalid: 0, + }, + ); +} + +#[test] +fn wycheproof_rsaes_pkcs1v15_4096_cases_48_to_51_match_expected_results() { + assert_rsaes_pkcs1v15_wycheproof_vectors( + RSAES_PKCS1_4096, + 4096, + 13..17, + ExpectedCounts { + valid: 4, + acceptable: 0, + invalid: 0, + }, + ); +} + +#[test] +fn wycheproof_rsaes_pkcs1v15_4096_cases_52_to_55_match_expected_results() { + assert_rsaes_pkcs1v15_wycheproof_vectors( + RSAES_PKCS1_4096, + 4096, + 17..21, + ExpectedCounts { + valid: 4, + acceptable: 0, + invalid: 0, + }, + ); +} + +#[test] +fn wycheproof_rsaes_pkcs1v15_4096_cases_56_to_59_match_expected_results() { + assert_rsaes_pkcs1v15_wycheproof_vectors( + RSAES_PKCS1_4096, + 4096, + 21..25, + ExpectedCounts { + valid: 3, + acceptable: 0, + invalid: 1, + }, + ); +} + +#[test] +fn wycheproof_rsaes_pkcs1v15_4096_cases_60_to_63_match_expected_results() { + assert_rsaes_pkcs1v15_wycheproof_vectors( + RSAES_PKCS1_4096, + 4096, + 25..29, + ExpectedCounts { + valid: 4, + acceptable: 0, + invalid: 0, + }, + ); +} + +#[test] +fn wycheproof_rsaes_pkcs1v15_4096_cases_64_to_67_match_expected_results() { + assert_rsaes_pkcs1v15_wycheproof_vectors( + RSAES_PKCS1_4096, + 4096, + 29..RSAES_PKCS1_GROUP_COUNT, + ExpectedCounts { + valid: 4, + acceptable: 0, + invalid: 0, }, ); } diff --git a/tests/xxh3_differential.rs b/tests/xxh3_differential.rs index 600084d4..1395af9e 100644 --- a/tests/xxh3_differential.rs +++ b/tests/xxh3_differential.rs @@ -1,4 +1,4 @@ -#![cfg(feature = "hashes")] +#![cfg(feature = "xxh3")] use proptest::prelude::*; use rscrypto::{ diff --git a/tools/ct-dudect/Cargo.lock b/tools/ct-dudect/Cargo.lock index cfda7259..89962947 100644 --- a/tools/ct-dudect/Cargo.lock +++ b/tools/ct-dudect/Cargo.lock @@ -30,9 +30,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "block2" @@ -51,9 +51,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" @@ -132,7 +132,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "block2", "libc", "objc2", @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "either" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] name = "getrandom" @@ -179,9 +179,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "nix" @@ -189,7 +189,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -221,18 +221,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -321,9 +321,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -396,18 +396,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", diff --git a/tools/ct-dudect/src/main.rs b/tools/ct-dudect/src/main.rs index 6170f4e3..b89d182c 100644 --- a/tools/ct-dudect/src/main.rs +++ b/tools/ct-dudect/src/main.rs @@ -46,6 +46,7 @@ const AEAD_PLAINTEXT: [u8; 44] = *b"constant-time seal buffer for key validation const RSA_PKCS1_2048: &str = include_str!("../../../testdata/rsa/wycheproof/rsa_pkcs1_2048_test.json"); const RSA_CT_KEY_A_INDEX: usize = 0; const RSA_CT_KEY_B_SAME_SHAPE_INDEX: usize = 2; +const RSA_CT_KEY_SHORT_CRT_EXPONENT_INDEX: usize = 21; fn samples() -> usize { std::env::var("RSCRYPTO_CT_DUDECT_SAMPLES") @@ -1460,6 +1461,34 @@ fn rsa_pss_fixed_vs_random_message(runner: &mut CtRunner, rng: &mut BenchRng) { } } +fn rsa_pkcs1v15_full_width_vs_short_canonical_crt_exponent(runner: &mut CtRunner, rng: &mut BenchRng) { + let full_width_key = rsa_ct_fixture_key(RSA_CT_KEY_A_INDEX); + let short_exponent_key = rsa_ct_fixture_key(RSA_CT_KEY_SHORT_CRT_EXPONENT_INDEX); + let full_width_blinding = rsa_blinding_pair(&full_width_key); + let short_exponent_blinding = rsa_blinding_pair(&short_exponent_key); + let message = [0x42; 32]; + + for class in balanced_classes(rng, samples()) { + let (key, (blinding_factor, blinding_inverse)) = if matches!(class, Class::Left) { + (&full_width_key, &full_width_blinding) + } else { + (&short_exponent_key, &short_exponent_blinding) + }; + runner.run_one(class, || { + let mut out = vec![0u8; key.signature_len()]; + key + .sign_pkcs1v15_with_blinding_factor( + RsaPkcs1v15Profile::Sha256, + &message, + blinding_factor, + blinding_inverse, + &mut out, + ) + .is_ok() + }); + } +} + fn rsa_oaep_decrypt_fixed_vs_random_plaintext(runner: &mut CtRunner, rng: &mut BenchRng) { let key = rsa_ct_fixture_key(RSA_CT_KEY_A_INDEX); let sig_len = key.signature_len(); @@ -2062,6 +2091,10 @@ ctbench_main_with_seeds!( (ecdsa_p384_diag_final_multiply_fixed_vs_random_secret, Some(0x703338346d756c73)), (rsa_pkcs1v15_fixed_vs_random_message, Some(0x7273615f7369676e)), (rsa_pss_fixed_vs_random_message, Some(0x7273615f70737373)), + ( + rsa_pkcs1v15_full_width_vs_short_canonical_crt_exponent, + Some(0x7273615f6372746c) + ), (rsa_oaep_decrypt_fixed_vs_random_plaintext, Some(0x7273615f6f616570)), (rsa_pkcs1v15_decrypt_fixed_vs_random_plaintext, Some(0x7273615f64656331)), (rsa_private_component_validation_fixed_vs_random_component, Some(0x7273615f636f6d70)), diff --git a/tools/ct-harness/Cargo.lock b/tools/ct-harness/Cargo.lock index 1270fca2..b37f5189 100644 --- a/tools/ct-harness/Cargo.lock +++ b/tools/ct-harness/Cargo.lock @@ -29,9 +29,9 @@ checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "either" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] name = "rayon" diff --git a/tools/wasm-runtime-vectors/Cargo.lock b/tools/wasm-runtime-vectors/Cargo.lock index 9cf586a1..290a790e 100644 --- a/tools/wasm-runtime-vectors/Cargo.lock +++ b/tools/wasm-runtime-vectors/Cargo.lock @@ -4,7 +4,7 @@ version = 4 [[package]] name = "rscrypto" -version = "0.6.4" +version = "0.7.8" [[package]] name = "rscrypto-wasm-runtime-vectors"