Shuffle before other plugins' collection hooks (#746)

Fixes #701.

Make pytest-randomly run first, even before other plugin’s collection
hooks marked "tryfirst", by turning the hook into a hook wrapper.

This ensures that pytest-django, and other plugins that group tests with
a stable sort, apply their grouping on top of the shuffled order, making
the final order reproducible from the seed.

Now we require pytest 8 and pluggy 1.0, the first version to support
hook wrappers.

Author: adamchainz

CHANGELOG.rst

@@ -7,6 +7,23 @@ Unreleased
 
 * Support Python 3.15.
 
+* Shuffle tests before other plugins’ ``pytest_collection_modifyitems`` hooks run.
+
+  Previously, pytest-randomly used a plain ``tryfirst`` hook, so another plugin using ``tryfirst``, such as pytest-django, could reorder tests before or after pytest-randomly’s shuffle, depending on the plugin registration order.
+  Since that order can vary between environments, this would make the final test order non-repeatable.
+
+  Now, pytest-randomly marks its hook as a hook wrapper, making it run before any other plugin’s ``pytest_collection_modifyitems`` hooks, regardless of registration order.
+  Plugins that group tests with a stable sort can apply their grouping on top of the shuffled order, making the final test order reproducible from the seed.
+
+  This is a low-impact breaking change because it you may not be able to reprduce a test order from the last pytest-randomly version by reusing a ``--randomly-seed`` value.
+
+  `PR #746 <https://github.com/pytest-dev/pytest-randomly/pull/746>`__.
+  Thanks to milssky for the report in `Issue #701 <https://github.com/pytest-dev/pytest-randomly/issues/701>`__.
+
+* Require pytest 8+, the first version to require a version of pluggy that supports hook wrappers, as used by the above fix.
+
+  `PR #746 <https://github.com/pytest-dev/pytest-randomly/pull/746>`__.
+
 * Reset `Polyfactory <https://polyfactory.litestar.dev/>`__\’s default random state at the start of every test, if it is installed.
 
   Thanks to Rahul Kumar in `PR #735 <https://github.com/pytest-dev/pytest-randomly/issues/735>`__.

pyproject.toml

@@ -37,7 +37,7 @@ classifiers = [
   "Typing :: Typed",
 ]
 dependencies = [
-  "pytest",
+  "pytest>=8",
 ]
 urls.Changelog = "https://github.com/pytest-dev/pytest-randomly/blob/main/CHANGELOG.rst"
 urls.Funding = "https://adamj.eu/books/"

src/pytest_randomly/__init__.py

@@ -2,7 +2,7 @@
 
 import argparse
 import random
-from collections.abc import Callable
+from collections.abc import Callable, Generator
 from functools import lru_cache
 from importlib.metadata import entry_points
 from itertools import groupby
@@ -205,11 +205,21 @@ def pytest_runtest_teardown(item: Item) -> None:
         _reseed(item.config, (_crc32(item.nodeid) + 1) % 2**32)
 
 
-@hookimpl(tryfirst=True)
-def pytest_collection_modifyitems(config: Config, items: list[Item]) -> None:
-    if not config.getoption("randomly_reorganize"):
-        return
+@hookimpl(wrapper=True, tryfirst=True)
+def pytest_collection_modifyitems(
+    config: Config, items: list[Item]
+) -> Generator[None, None, None]:
+    # Reorganize within this wrapper, before the yield, so it runs before all
+    # non-wrapper implementations of this hook, regardless of plugin
+    # registration order. Plugins that group tests with a stable sort, like
+    # pytest-django, then apply their grouping on top of the shuffled order,
+    # making the final order reproducible from the seed.
+    if config.getoption("randomly_reorganize"):
+        _reorganize_items(config, items)
+    return (yield)
 
+
+def _reorganize_items(config: Config, items: list[Item]) -> None:
     seed = _reseed(config)
 
     modules_items: list[tuple[ModuleType | None, list[Item]]] = []

tests/test_pytest_randomly.py

@@ -409,6 +409,50 @@ def test_d():
     ]
 
 
+def test_reordered_before_other_tryfirst_hook(ourtester):
+    """
+    The reorganization should run before other plugins’
+    pytest_collection_modifyitems hooks, even tryfirst ones registered after
+    pytest-randomly, so plugins that group tests with a stable sort, like
+    pytest-django, preserve the shuffled order within their groups.
+    """
+    ourtester.makeconftest(
+        """
+        import pytest
+
+        @pytest.hookimpl(tryfirst=True)
+        def pytest_collection_modifyitems(items):
+            items.sort(key=lambda item: 0 if "db" in item.name else 1)
+        """
+    )
+    ourtester.makepyfile(
+        test_one="""
+        def test_db_a():
+            pass
+
+        def test_db_b():
+            pass
+
+        def test_plain_c():
+            pass
+
+        def test_plain_d():
+            pass
+        """
+    )
+    args = ["-v", "--randomly-seed=15"]
+
+    out = ourtester.runpytest(*args)
+
+    out.assert_outcomes(passed=4, failed=0)
+    assert out.outlines[9:13] == [
+        "test_one.py::test_db_a PASSED",
+        "test_one.py::test_db_b PASSED",
+        "test_one.py::test_plain_d PASSED",
+        "test_one.py::test_plain_c PASSED",
+    ]
+
+
 def test_doctests_reordered(ourtester):
     ourtester.makepyfile(
         test_one="""