Add --rerun-warning flag to emit a warning on each scheduled rerun

README.rst

@@ -334,6 +334,22 @@ Each retried attempt's traceback is appended to the ``rerun test summary
 info`` section. The section is emitted automatically when the flag is set,
 so ``-rR`` is not required.
 
+Warn when a test is rerun
+-------------------------
+
+To make reruns more visible on CI, pass ``--rerun-warning``. Each time a
+test is scheduled for rerun a ``PytestWarning`` is emitted, which tools
+like `pytest-github-actions-annotate-failures
+<https://pypi.org/project/pytest-github-actions-annotate-failures/>`_ can
+turn into annotations:
+
+.. code-block:: bash
+
+   $ pytest --reruns 2 --rerun-warning
+
+``filterwarnings = error`` does not turn these warnings into errors, so they
+never fail a test or abort the run.
+
 Output
 ------
 

changes/318.feature.rst

@@ -0,0 +1,3 @@
+Add ``--rerun-warning`` flag that emits a ``PytestWarning`` each time a test
+is scheduled for rerun, to surface flaky tests in CI (e.g. via
+pytest-github-actions-annotate-failures annotations).

src/pytest_rerunfailures.py

@@ -178,6 +178,15 @@ def pytest_addoption(parser):
         "'rerun test summary info' section, which is emitted automatically "
         "when this flag is set.",
     )
+    group._addoption(
+        "--rerun-warning",
+        action="store_true",
+        dest="rerun_warning",
+        help="Emit a PytestWarning each time a test is scheduled for rerun. "
+        "Useful to surface flaky tests in CI, e.g. via annotations from "
+        "pytest-github-actions-annotate-failures. filterwarnings=error "
+        "does not turn these warnings into errors.",
+    )
     group.addoption(
         "--max-suite-reruns",
         action="store",
@@ -280,6 +289,37 @@ def _warn_pdb_disables_reruns(config, item=None):
             )
 
 
+def _warn_rerun(config, nodeid, attempt, item=None):
+    """Warn that a test failed and will be rerun (``--rerun-warning``).
+
+    As in ``_warn_pdb_disables_reruns``, ``-W error`` must not escalate the
+    warning into an INTERNALERROR, so an escalated warning is recorded through
+    pytest's warning hook instead. Without an item (a crashed test rescheduled
+    by the xdist controller) the warning is always recorded that way.
+    """
+    warning = pytest.PytestWarning(
+        f"{nodeid} failed on attempt {attempt} and will be rerun"
+    )
+    if item is not None:
+        try:
+            item.warn(warning)
+            return
+        except Warning:
+            filename, lineno = str(item.path), (item.location[1] or 0) + 1
+    else:
+        filename, lineno = nodeid.split("::")[0], 0
+    config.hook.pytest_warning_recorded.call_historic(
+        kwargs=dict(
+            warning_message=warnings.WarningMessage(
+                warning, type(warning), filename, lineno
+            ),
+            when="runtest",
+            nodeid=nodeid,
+            location=None,
+        )
+    )
+
+
 def get_reruns_count(item):
     reruns = item.session.config.getoption("force_reruns")
     if reruns is not None:
@@ -821,6 +861,10 @@ def pytest_handlecrashitem(self, crashitem, report, sched):
             try:
                 sched.mark_test_pending(crashitem)
                 report.outcome = "rerun"
+                if sched.config.option.rerun_warning:
+                    _warn_rerun(
+                        sched.config, crashitem, db.get_test_failures(crashitem) + 1
+                    )
             except NotImplementedError:
                 if reserved_suite_rerun:
                     db.decrement_suite_reruns()
@@ -1337,6 +1381,10 @@ def pytest_runtest_protocol(item, nextitem):
                         continue
 
                 report.outcome = "rerun"
+                if item.session.config.option.rerun_warning:
+                    _warn_rerun(
+                        item.config, item.nodeid, item.execution_count, item=item
+                    )
                 time.sleep(delay * delay_backoff_factor ** (item.execution_count - 1))
 
                 if not parallel or works_with_current_xdist():

tests/test_pytest_rerunfailures.py

@@ -798,6 +798,66 @@ def test_skips(bad_teardown):
     ]
 
 
+def test_rerun_warning_emitted(testdir):
+    testdir.makepyfile(
+        f"""
+        def test_pass():
+            {temporary_failure()}"""
+    )
+    result = testdir.runpytest("--reruns", "1", "--rerun-warning")
+    assert_outcomes(result, passed=1, rerun=1)
+    result.stdout.fnmatch_lines_random([
+        "*PytestWarning: *test_pass* failed on attempt 1 and will be rerun*"
+    ])
+
+
+def test_rerun_warning_with_warnings_as_errors(testdir):
+    """`-W error` must not escalate the warning into an INTERNALERROR."""
+    testdir.makepyfile(
+        f"""
+        def test_pass():
+            {temporary_failure()}
+
+        def test_other():
+            pass"""
+    )
+    result = testdir.runpytest("-W", "error", "--reruns", "1", "--rerun-warning")
+    assert_outcomes(result, passed=2, rerun=1)
+    result.stdout.no_fnmatch_line("INTERNALERROR*")
+    result.stdout.fnmatch_lines_random([
+        "*PytestWarning: *test_pass* failed on attempt 1 and will be rerun*"
+    ])
+
+
+@pytest.mark.skipif(not has_xdist, reason="requires xdist with crashitem")
+def test_rerun_warning_emitted_for_temporary_test_crash(testdir):
+    testdir.makepyfile(
+        f"""
+        def test_crash():
+            {temporary_crash()}
+
+        def test_pass():
+            pass"""
+    )
+    result = testdir.runpytest(
+        "-p", "xdist", "-n", "1", "--reruns", "1", "--rerun-warning"
+    )
+    assert_outcomes(result, passed=2, rerun=1)
+    result.stdout.fnmatch_lines_random([
+        "*PytestWarning: *test_crash failed on attempt 1 and will be rerun*"
+    ])
+
+
+def test_no_rerun_warning_by_default(testdir):
+    testdir.makepyfile(
+        f"""
+        def test_pass():
+            {temporary_failure()}"""
+    )
+    result = testdir.runpytest("--reruns", "1")
+    assert "will be rerun" not in result.stdout.str()
+
+
 @pytest.mark.parametrize("report_flag", ["-ra", "-rA"])
 def test_extra_test_summary_for_reruns_with_reportchars_all(testdir, report_flag):
     testdir.makepyfile(