Repository navigation
Fix Linux wheel illegal-instruction crashes #511
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
cjacoby-sptfy marked this conversation as resolved.
|
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| #!/usr/bin/env python3 | ||
| """Smoke-test an installed Linux wheel across its x86_64 CPU compatibility boundary.""" | ||
|
|
||
| import argparse | ||
| import signal | ||
| import subprocess | ||
| import sys | ||
| import tempfile | ||
|
|
||
|
|
||
| SUPPORTED_CPU_MODELS = ("IvyBridge", "EPYC-Milan") | ||
| UNSUPPORTED_CPU_MODELS = ("Nehalem",) | ||
| DEFAULT_CPU_MODELS = SUPPORTED_CPU_MODELS + UNSUPPORTED_CPU_MODELS | ||
| SMOKE_TEST_TIMEOUT_SECONDS = 60 | ||
| RUNTIME_PROBE = "print('Python runtime started successfully')" | ||
| WHEEL_SMOKE_TEST = ( | ||
| "import numpy as np; " | ||
| "from pedalboard import Gain; " | ||
| "output = Gain(gain_db=-6)(np.zeros((1, 1024), dtype=np.float32), 48000); " | ||
| "assert output.shape == (1, 1024)" | ||
| ) | ||
|
|
||
|
|
||
| def run_on_cpu( | ||
| cpu_model: str, python_code: str, python: str = sys.executable | ||
| ) -> subprocess.CompletedProcess[str]: | ||
| with tempfile.TemporaryDirectory() as working_directory: | ||
| return subprocess.run( | ||
| [ | ||
| "qemu-x86_64", | ||
| "-cpu", | ||
| cpu_model, | ||
| python, | ||
| "-c", | ||
| python_code, | ||
| ], | ||
| capture_output=True, | ||
| cwd=working_directory, | ||
| text=True, | ||
| timeout=SMOKE_TEST_TIMEOUT_SECONDS, | ||
| ) | ||
|
|
||
|
|
||
| def format_failure(result: subprocess.CompletedProcess[str]) -> str: | ||
| return f"exit code {result.returncode}\n{result.stdout}{result.stderr}" | ||
|
|
||
|
|
||
| def smoke_test_wheel(cpu_model: str, python: str = sys.executable) -> None: | ||
| print(f"Testing installed wheel on {cpu_model}...") | ||
| if cpu_model in UNSUPPORTED_CPU_MODELS: | ||
| runtime_probe = run_on_cpu(cpu_model, RUNTIME_PROBE, python) | ||
| if runtime_probe.returncode != 0: | ||
| raise RuntimeError( | ||
| f"Python runtime failed to start on {cpu_model} ({format_failure(runtime_probe)})" | ||
| ) | ||
|
|
||
| result = run_on_cpu(cpu_model, WHEEL_SMOKE_TEST, python) | ||
| if cpu_model in UNSUPPORTED_CPU_MODELS: | ||
| expected_returncode = -signal.SIGILL | ||
| if result.returncode != expected_returncode: | ||
| raise RuntimeError( | ||
| f"pedalboard wheel unexpectedly ran on non-AVX CPU {cpu_model}; " | ||
| f"expected SIGILL ({expected_returncode}), got {format_failure(result)}" | ||
| ) | ||
| print(f"Installed wheel correctly requires AVX on {cpu_model}.") | ||
| return | ||
|
|
||
| if result.returncode != 0: | ||
| raise RuntimeError( | ||
| f"pedalboard wheel failed its smoke test on {cpu_model} ({format_failure(result)})" | ||
| ) | ||
| print(f"Installed wheel passed on {cpu_model}.") | ||
|
|
||
|
|
||
| def main() -> None: | ||
| parser = argparse.ArgumentParser() | ||
| parser.add_argument("cpu_models", nargs="*", default=DEFAULT_CPU_MODELS) | ||
| args = parser.parse_args() | ||
|
|
||
| for cpu_model in args.cpu_models: | ||
| smoke_test_wheel(cpu_model) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| import os | ||
| from pathlib import Path | ||
| import subprocess | ||
| import sys | ||
|
|
||
| import pytest | ||
|
|
||
|
|
||
| pytestmark = pytest.mark.skipif( | ||
| sys.platform == "win32", reason="The wheel compatibility harness runs only on Linux" | ||
| ) | ||
|
|
||
|
|
||
| HARNESS = Path(__file__).parents[1] / "scripts" / "test_linux_wheel_cpu_compatibility.py" | ||
| NON_AVX_CPU = "Nehalem" | ||
|
|
||
|
|
||
| def run_harness( | ||
| tmp_path: Path, | ||
| emulator_body: str, | ||
| cpu_models: tuple[str, ...] = (NON_AVX_CPU,), | ||
| ) -> tuple[subprocess.CompletedProcess, list[str]]: | ||
| invocation_log = tmp_path / "qemu-invocations" | ||
| emulator = tmp_path / "qemu-x86_64" | ||
| emulator.write_text( | ||
| "#!/bin/sh\n" | ||
| 'cpu_model="$2"\n' | ||
| 'for argument do code="$argument"; done\n' | ||
| f'printf "%s|%s\\n" "$cpu_model" "$code" >> "{invocation_log}"\n' | ||
| f"{emulator_body}\n" | ||
| ) | ||
| emulator.chmod(0o755) | ||
|
|
||
| env = os.environ.copy() | ||
| env["PATH"] = f"{tmp_path}{os.pathsep}{env['PATH']}" | ||
| result = subprocess.run( | ||
| [sys.executable, HARNESS, *cpu_models], | ||
| capture_output=True, | ||
| env=env, | ||
| text=True, | ||
| ) | ||
| invocations = invocation_log.read_text().splitlines() | ||
| return result, invocations | ||
|
|
||
|
|
||
| def test_non_avx_cpu_requires_wheel_import_to_terminate_with_sigill(tmp_path: Path) -> None: | ||
| result, invocations = run_harness( | ||
| tmp_path, | ||
| 'case "$code" in *pedalboard*) kill -ILL $$ ;; *) exit 0 ;; esac', | ||
| ) | ||
|
|
||
| assert result.returncode == 0, result.stderr | ||
| assert len(invocations) == 2 | ||
| assert "pedalboard" not in invocations[0] | ||
| assert "pedalboard" in invocations[1] | ||
|
|
||
|
|
||
| def test_default_cpu_models_include_the_non_avx_boundary_check(tmp_path: Path) -> None: | ||
| result, invocations = run_harness( | ||
| tmp_path, | ||
| 'case "$cpu_model:$code" in Nehalem:*pedalboard*) kill -ILL $$ ;; *) exit 0 ;; esac', | ||
| cpu_models=(), | ||
| ) | ||
|
|
||
| assert result.returncode == 0, result.stderr | ||
| assert any(invocation.startswith("IvyBridge|") for invocation in invocations) | ||
| assert any(invocation.startswith("EPYC-Milan|") for invocation in invocations) | ||
| non_avx_invocations = [ | ||
| invocation for invocation in invocations if invocation.startswith(f"{NON_AVX_CPU}|") | ||
| ] | ||
| assert len(non_avx_invocations) == 2 | ||
| assert "pedalboard" not in non_avx_invocations[0] | ||
| assert "pedalboard" in non_avx_invocations[1] | ||
|
|
||
|
|
||
| def test_non_avx_cpu_rejects_a_broken_emulator_or_python_runtime(tmp_path: Path) -> None: | ||
| result, invocations = run_harness(tmp_path, "kill -ILL $$") | ||
|
|
||
| assert result.returncode != 0 | ||
| assert len(invocations) == 1 | ||
| assert "pedalboard" not in invocations[0] | ||
|
|
||
|
|
||
| def test_non_avx_cpu_rejects_a_non_sigill_wheel_failure(tmp_path: Path) -> None: | ||
| result, invocations = run_harness( | ||
| tmp_path, | ||
| 'case "$code" in *pedalboard*) exit 1 ;; *) exit 0 ;; esac', | ||
| ) | ||
|
|
||
| assert result.returncode != 0 | ||
| assert len(invocations) == 2 | ||
| assert "pedalboard" not in invocations[0] | ||
| assert "pedalboard" in invocations[1] |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.