Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/copilot-instructions.md
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,10 @@ The project uses GitHub Actions with maturin for cross-platform wheel building:
emulated CPU does not report the expected features
- `pool-sanitizers`: the scan thread pool's tests (`par::`) under ThreadSanitizer and Miri
with `HEXHAMMING_NUM_THREADS=4`
- `PGO wheels` (`pgo-wheels.yml`): profile-guided wheels for Linux x86_64, macOS arm64 and
Windows, trained with `scripts/pgo_train.py`. Releases call it on tags; on a pull request it
runs only with the `ci:wheels` label and reports PGO vs. plain timings from
`scripts/pgo_bench.py` in the job summary. Nothing is published from pull requests
- `Benchmark PR` (`benchmark.yml`): benchmarks every PR against its base branch on x86 and
comments the comparison
- Builds wheels for Linux (manylinux), macOS, Windows
Expand Down
163 changes: 163 additions & 0 deletions .github/workflows/pgo-wheels.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
name: PGO wheels

# Profile-guided optimization for the wheels built natively on their runner:
# build an instrumented wheel, train it with scripts/pgo_train.py, merge the
# profile, and rebuild every wheel with it. The release workflow calls this on
# tags. On pull requests it runs only with the `ci:wheels` label, validating
# the build and reporting PGO vs. plain timings; nothing is published there.

on:
workflow_call:
pull_request:
types: [opened, synchronize, reopened, labeled]

permissions:
contents: read

jobs:
pgo-wheels:
if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:wheels')
name: PGO wheels (${{ matrix.platform }})
runs-on: ${{ matrix.os }}
timeout-minutes: 90
strategy:
# Report every platform; publishing still waits for all of them.
fail-fast: false
matrix:
include:
- platform: linux-x86_64
os: ubuntu-latest
target: x86_64
manylinux: auto
interpreters: --find-interpreter
artifact: wheels-linux-x86_64
# macOS and Windows build for the runner's default Python, as the
# release jobs did before PGO.
- platform: macos-aarch64
os: macos-latest
target: aarch64-apple-darwin
manylinux: ""
interpreters: ""
artifact: wheels-macos-aarch64
- platform: windows-x86_64
os: windows-latest
target: x86_64-pc-windows-msvc
manylinux: ""
interpreters: ""
artifact: wheels-windows
defaults:
run:
shell: bash
env:
PGO_DATA: ${{ github.workspace }}/pgo-data
PGO_PROFILE: ${{ github.workspace }}/pgo.profdata

steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

- name: Install Rust
uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
with:
toolchain: stable
components: llvm-tools

# llvm-profdata must match the LLVM that compiles the wheels, so the
# manylinux container installs this exact toolchain version too.
- name: Resolve toolchain and training interpreter
id: env
run: |
host=$(rustc -vV | sed -n 's/^host: //p')
# The interpreter maturin uses by default. On Windows `python3` can
# be the Microsoft Store stub, so use `python` there.
if [ "$RUNNER_OS" = Windows ]; then python=$(command -v python); else python=$(command -v python3); fi
version=$("$python" -c 'import sys; print("%d.%d" % sys.version_info[:2])')
"$python" -m venv "$RUNNER_TEMP/train"
"$python" -m venv "$RUNNER_TEMP/plain"
if [ "$RUNNER_OS" = Windows ]; then bin=Scripts/python.exe; else bin=bin/python; fi
{
echo "rust=$(rustc --version | cut -d' ' -f2)"
echo "profdata=$(rustc --print sysroot)/lib/rustlib/$host/bin/llvm-profdata"
echo "python-version=$version"
echo "abi=cp${version/./}"
echo "train-python=$RUNNER_TEMP/train/$bin"
echo "plain-python=$RUNNER_TEMP/plain/$bin"
} >> "$GITHUB_OUTPUT"
echo "Training with Python $version and $(rustc --version)"

- name: Build instrumented wheel
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1
env:
RUSTFLAGS: -Cprofile-generate=${{ env.PGO_DATA }}
with:
target: ${{ matrix.target }}
manylinux: ${{ matrix.manylinux }}
args: --release --out instrumented --interpreter ${{ steps.env.outputs.python-version }}
before-script-linux: |
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain ${{ steps.env.outputs.rust }}
source $HOME/.cargo/env

- name: Train
run: |
py="${{ steps.env.outputs.train-python }}"
"$py" -m pip install --quiet --force-reinstall --no-deps instrumented/*.whl
"$py" scripts/pgo_train.py

- name: Merge profile
run: '"${{ steps.env.outputs.profdata }}" merge -o "$PGO_PROFILE" "$PGO_DATA"'

- name: Build PGO wheels
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1
env:
# PGSO would size-optimize code the training never ran; keep it at
# full optimization so untrained paths do not regress.
RUSTFLAGS: -Cprofile-use=${{ env.PGO_PROFILE }} -Cllvm-args=-pgso=false
with:
target: ${{ matrix.target }}
manylinux: ${{ matrix.manylinux }}
args: --release --out dist ${{ matrix.interpreters }}
before-script-linux: |
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain ${{ steps.env.outputs.rust }}
source $HOME/.cargo/env

- name: Test PGO wheel
run: |
py="${{ steps.env.outputs.train-python }}"
wheels=(dist/*-${{ steps.env.outputs.abi }}-${{ steps.env.outputs.abi }}-*.whl)
"$py" -m pip install --quiet pytest
"$py" -m pip install --quiet --force-reinstall --no-deps "${wheels[0]}"
"$py" -m pytest -q -p no:cacheprovider test/ -k "not bench"

- name: Build plain wheel for comparison
if: github.event_name == 'pull_request'
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1
with:
target: ${{ matrix.target }}
manylinux: ${{ matrix.manylinux }}
args: --release --out plain --interpreter ${{ steps.env.outputs.python-version }}
before-script-linux: |
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain ${{ steps.env.outputs.rust }}
source $HOME/.cargo/env

- name: Compare PGO and plain builds
if: github.event_name == 'pull_request'
run: |
pgo="${{ steps.env.outputs.train-python }}"
plain="${{ steps.env.outputs.plain-python }}"
"$plain" -m pip install --quiet --force-reinstall --no-deps plain/*.whl
mkdir -p timings
for run in 1 2 3 4 5; do
"$plain" scripts/pgo_bench.py > "timings/plain-$run.json"
"$pgo" scripts/pgo_bench.py > "timings/pgo-$run.json"
done
{
echo "### PGO vs. plain (${{ matrix.platform }})"
echo
"$pgo" scripts/pgo_bench.py --compare timings/plain-*.json vs timings/pgo-*.json
} | tee -a "$GITHUB_STEP_SUMMARY"

- name: Upload wheels
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ${{ matrix.artifact }}
path: dist/*.whl
if-no-files-found: error
38 changes: 13 additions & 25 deletions .github/workflows/pythonpackage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,15 @@ jobs:
name: wheels-sdist
path: dist/*.tar.gz

# Wheels built natively on their runner (Linux x86_64, macOS arm64,
# Windows) are PGO builds; see pgo-wheels.yml.
wheels_pgo:
if: startsWith(github.ref, 'refs/tags')
needs: build-and-test
name: Build PGO wheels
uses: ./.github/workflows/pgo-wheels.yml

# Cross-built wheels cannot run the PGO training on the runner.
wheels_linux:
if: startsWith(github.ref, 'refs/tags')
needs: build-and-test
Expand All @@ -244,7 +253,7 @@ jobs:
strategy:
fail-fast: true
matrix:
target: [x86_64, aarch64]
target: [aarch64]

steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
Expand Down Expand Up @@ -281,15 +290,15 @@ jobs:
strategy:
fail-fast: true
matrix:
target: [x86_64, aarch64]
target: [x86_64]

steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

- name: Build wheels
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1
with:
target: ${{ matrix.target == 'aarch64' && 'aarch64-apple-darwin' || 'x86_64-apple-darwin' }}
target: x86_64-apple-darwin
args: --release --out dist

- name: Upload wheels
Expand All @@ -299,30 +308,9 @@ jobs:
path: dist/*.whl
if-no-files-found: error

wheels_windows:
if: startsWith(github.ref, 'refs/tags')
needs: build-and-test
name: Build Windows wheels
runs-on: windows-latest

steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

- name: Build wheels
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1
with:
args: --release --out dist

- name: Upload wheels
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: wheels-windows
path: dist/*.whl
if-no-files-found: error

publish-wheels:
if: startsWith(github.ref, 'refs/tags')
needs: [sdist, wheels_linux, wheels_macos, wheels_windows]
needs: [sdist, wheels_pgo, wheels_linux, wheels_macos]
name: Publish wheels
runs-on: ubuntu-latest
environment: pypi
Expand Down
113 changes: 113 additions & 0 deletions scripts/pgo_bench.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
"""Time representative hexhamming calls and print them as JSON (ns per call).

Used by the PGO wheel workflow to compare a PGO build with a plain build on
the same runner. Each case reports the best of several rounds of repeated
calls.

Separate processes vary by several percent (frequency, code placement), so
run each build a few times, alternating, and compare the per-case minimum.

Usage: python scripts/pgo_bench.py [module] > results.json
python scripts/pgo_bench.py --compare plain-*.json vs pgo-*.json
"""

import argparse
import importlib
import json
import random
import time


def cases(h):
rng = random.Random(99)
s16, t16 = rng.randbytes(8).hex(), rng.randbytes(8).hex()
s64, t64 = rng.randbytes(32).hex(), rng.randbytes(32).hex()
s1k, t1k = rng.randbytes(512).hex(), rng.randbytes(512).hex()
b16, c16 = rng.randbytes(16), rng.randbytes(16)
b64, c64 = rng.randbytes(64), rng.randbytes(64)
cat16 = rng.randbytes(16 * 1024)
cat64 = rng.randbytes(64 * 16384)
pa, pb = rng.randbytes(16 * 64), rng.randbytes(16 * 64)
return [
("hamming_distance_string/16", h.hamming_distance_string, (s16, t16)),
("hamming_distance_string/64", h.hamming_distance_string, (s64, t64)),
("hamming_distance_string/1024", h.hamming_distance_string, (s1k, t1k)),
("hamming_distance_bytes/16", h.hamming_distance_bytes, (b16, c16)),
("hamming_distance_bytes/64", h.hamming_distance_bytes, (b64, c64)),
(
"check_hexstrings_within_dist/16",
h.check_hexstrings_within_dist,
(s16, t16, 10),
),
("check_bytes_within_dist/16", h.check_bytes_within_dist, (b16, c16, 10)),
("best_within/1024x16", h.check_bytes_arrays_best_within_dist, (cat16, b16, 0)),
("all_within/1024x16", h.check_bytes_arrays_all_within_dist, (cat16, b16, 0)),
(
"best_within/16384x64",
h.check_bytes_arrays_best_within_dist,
(cat64, b64, 0),
),
(
"hamming_distances_bytes_packed/64x16",
h.hamming_distances_bytes_packed,
(pa, pb, 16),
),
]


def measure(fn, args, rounds=7, budget=0.05):
calls = 16
while True:
start = time.perf_counter()
for _ in range(calls):
fn(*args)
if time.perf_counter() - start > budget / 10:
break
calls *= 4
calls = max(1, int(calls * budget / max(time.perf_counter() - start, 1e-9) / 10))
best = float("inf")
for _ in range(rounds):
start = time.perf_counter_ns()
for _ in range(calls):
fn(*args)
best = min(best, (time.perf_counter_ns() - start) / calls)
return best


def load_min(paths):
best = {}
for path in paths:
with open(path) as f:
for name, value in json.load(f).items():
best[name] = min(value, best.get(name, float("inf")))
return best


def compare(paths):
split = paths.index("vs")
plain, pgo = load_min(paths[:split]), load_min(paths[split + 1 :])
print(f"Best of {split} plain and {len(paths) - split - 1} PGO runs.")
print()
print("| Case | plain (ns) | PGO (ns) | speedup |")
print("|---|---:|---:|---:|")
for name, base in plain.items():
new = pgo[name]
print(f"| `{name}` | {base:,.1f} | {new:,.1f} | {base / new:.2f}x |")


def main():
parser = argparse.ArgumentParser()
parser.add_argument("module", nargs="?", default="hexhamming")
parser.add_argument("--compare", nargs=argparse.REMAINDER)
args = parser.parse_args()
if args.compare:
compare(args.compare)
return
h = importlib.import_module(args.module)
print(
json.dumps({name: measure(fn, fargs) for name, fn, fargs in cases(h)}, indent=1)
)


if __name__ == "__main__":
main()
Loading
Loading