Fix rerun filtering for teardown exceptions

changes/261.bugfix.rst

@@ -0,0 +1 @@
+Report a test as rerun when a teardown exception matches the rerun filter after an earlier test-phase failure.

changes/385.bugfix.rst

@@ -0,0 +1 @@
+Preserve all failed test phases when a later teardown failure triggers a rerun.

src/pytest_rerunfailures.py

@@ -731,22 +731,6 @@ def _should_hard_fail_on_error(item, report, excinfo):
         return (not matches_rerun_only) or matches_rerun_except
 
 
-def _should_not_rerun(item, report, reruns, condition):
-    xfail = hasattr(report, "wasxfail")
-    is_terminal_error = any(item._terminal_errors.values())
-    has_failed_subtests = report.when == "call" and _get_num_failed_subtests(item) > 0
-
-    if (
-        item.execution_count > reruns
-        or (not report.failed and not has_failed_subtests)
-        or xfail
-        or is_terminal_error
-    ):
-        return True
-
-    return not condition
-
-
 def is_master(config):
     return not (hasattr(config, "workerinput") or hasattr(config, "slaveinput"))
 
@@ -1116,8 +1100,11 @@ def _get_reruns_condition_failures(item):
     return failures
 
 
-def _reruns_condition_matches_phase(item, phase):
-    """Return whether a failed phase matched the attempt's condition."""
+def _rerun_matches_phase(item, phase):
+    """Return whether a failed phase can trigger a rerun."""
+    if item._terminal_errors.get(phase, False):
+        return False
+
     rerun_marker = _get_marker(item)
     if rerun_marker is None or "condition" not in rerun_marker.kwargs:
         return True
@@ -1129,6 +1116,36 @@ def _reruns_condition_matches_phase(item, phase):
     )
 
 
+def _has_rerun_except_terminal_error(item):
+    """Return whether an exception explicitly excluded reruns this attempt."""
+    rerun_except_errors = _get_rerun_filter_regex(item, "rerun_except")
+    if not rerun_except_errors:
+        return False
+
+    return any(
+        _matches_any_rerun_except_error(rerun_except_errors, excinfo)
+        for phase_excinfos in item._rerun_condition_excinfos.values()
+        for excinfo in phase_excinfos
+    )
+
+
+def _should_rerun(item, reruns, condition):
+    """Return whether this attempt should be rerun as a whole."""
+    if (
+        not condition
+        or item.execution_count > reruns
+        or _has_rerun_except_terminal_error(item)
+        or item._test_xfailed.get("setup", False)
+        or item._test_xfailed.get("call", False)
+    ):
+        return False
+
+    return any(
+        _rerun_matches_phase(item, phase)
+        for phase, _, _ in _get_reruns_condition_failures(item)
+    )
+
+
 def _teardown_suspended_finalizers(item, call, report):
     """Tear down the scopes held back for a re-run that will not happen.
 
@@ -1303,56 +1320,51 @@ def pytest_runtest_protocol(item, nextitem):
         item.ihook.pytest_runtest_logstart(nodeid=item.nodeid, location=item.location)
         reports = runtestprotocol(item, nextitem=nextitem, log=False)
 
-        condition = get_reruns_condition(item, _get_reruns_condition_failures(item))
-        rerun_triggered = False
+        condition_failures = _get_reruns_condition_failures(item)
+        condition = get_reruns_condition(item, condition_failures)
+        rerun_triggered = _should_rerun(item, reruns, condition)
+
+        if rerun_triggered:
+            max_suite_reruns = item.session.config.option.max_suite_reruns
+            if max_suite_reruns is not None and not db.try_increment_suite_reruns(
+                max_suite_reruns
+            ):
+                # Suite-wide limit exhausted -- log all reports as final results.
+                _restore_suspended_finalizers(item)
+                rerun_triggered = False
+
+        rerun_report = next(
+            (
+                report
+                for report in reports
+                if report.when == "call" and _get_num_failed_subtests(item) > 0
+            ),
+            next((report for report in reports if report.failed), None),
+        )
         for report in reports:  # 3 reports: setup, call, teardown
             report.rerun = item.execution_count - 1
-            if rerun_triggered:
-                if report.failed:
-                    report.outcome = "rerun"
-                item.ihook.pytest_runtest_logreport(report=report)
-            elif (
-                condition
-                and not _reruns_condition_matches_phase(item, report.when)
-                and (
-                    report.failed
-                    or (report.when == "call" and _get_num_failed_subtests(item) > 0)
-                )
+            if rerun_triggered and (
+                report.failed
+                or (report.when == "call" and _get_num_failed_subtests(item) > 0)
             ):
-                # Another failed phase matched the condition and will carry
-                # this intermediate attempt's rerun report. Do not publish a
-                # nonmatching failure as a final result first.
-                continue
-            elif _should_not_rerun(item, report, reruns, condition):
-                # no rerun needed or one already triggered, log normally
-                item.ihook.pytest_runtest_logreport(report=report)
-            else:
-                # failure detected and reruns not exhausted, since i < reruns
-                max_suite_reruns = item.session.config.option.max_suite_reruns
-                if max_suite_reruns is not None:
-                    if not db.try_increment_suite_reruns(max_suite_reruns):
-                        # Suite-wide limit exhausted -- log as final failure.
-                        _restore_suspended_finalizers(item)
-                        item.ihook.pytest_runtest_logreport(report=report)
-                        continue
-
                 report.outcome = "rerun"
-                time.sleep(delay * delay_backoff_factor ** (item.execution_count - 1))
-
                 if not parallel or works_with_current_xdist():
-                    # will rerun test, log intermediate result
                     item.ihook.pytest_runtest_logreport(report=report)
+            else:
+                item.ihook.pytest_runtest_logreport(report=report)
 
-                # cleanin item's cashed results from any level of setups
-                _remove_cached_results_from_failed_fixtures(item)
-                _remove_failed_setup_state_from_session(item)
-                _discard_test_class_instance(item)
-                _remove_failed_subtests_from_report(item, report)
-                _remove_failed_subtest_reports_from_stats(
-                    item.config, item.session, item.nodeid
-                )
+        if rerun_triggered:
+            time.sleep(delay * delay_backoff_factor ** (item.execution_count - 1))
 
-                rerun_triggered = True
+            # Clean cached results from any level of setups.
+            _remove_cached_results_from_failed_fixtures(item)
+            _remove_failed_setup_state_from_session(item)
+            _discard_test_class_instance(item)
+            if rerun_report is not None:
+                _remove_failed_subtests_from_report(item, rerun_report)
+            _remove_failed_subtest_reports_from_stats(
+                item.config, item.session, item.nodeid
+            )
 
         # Do not retain ExceptionInfo tracebacks and their frame locals for the
         # lifetime of the collected item/session.

tests/test_pytest_rerunfailures.py

@@ -2313,7 +2313,7 @@ def test_service(service):
 
     result = testdir.runpytest()
     assert result.ret == 0
-    assert_outcomes(result, passed=1, rerun=1)
+    assert_outcomes(result, passed=1, rerun=2)
 
 
 def test_condition_exception_state_is_released_after_attempt(testdir):
@@ -2616,6 +2616,33 @@ def test_fail(broken_fixture):
     assert result.stdout.str().count("module teardown") == 1
 
 
+def test_teardown_error_can_trigger_only_rerun(testdir):
+    testdir.makepyfile(
+        """
+        import pytest
+
+        attempts = 0
+
+        @pytest.fixture
+        def broken_fixture():
+            yield
+            if attempts == 1:
+                raise ValueError("teardown error")
+
+        @pytest.mark.flaky(reruns=1, only_rerun=["ValueError"])
+        def test_fail(broken_fixture):
+            global attempts
+            attempts += 1
+            if attempts == 1:
+                raise AssertionError("call error")
+        """
+    )
+
+    result = testdir.runpytest()
+
+    assert_outcomes(result, passed=1, rerun=2)
+
+
 @pytest.mark.parametrize(
     "outcome,skipped,xfailed",
     [("skip", 3, 0), ("xfail", 0, 3)],
@@ -3890,3 +3917,70 @@ def test_fail():
     )
     result = testdir.runpytest("-s", "--reruns", "1", "--max-suite-reruns", "0")
     result.stdout.fnmatch_lines("*module teardown*")
+
+
+def test_nonmatching_call_failure_is_preserved_on_last_rerun(testdir):
+    testdir.makepyfile(
+        """
+        import pytest
+
+        @pytest.fixture
+        def teardown_error():
+            yield
+            raise ValueError("teardown error")
+
+        @pytest.mark.flaky(reruns=1, only_rerun=["ValueError"])
+        def test_fail(teardown_error):
+            raise AssertionError("call error")
+        """
+    )
+    result = testdir.runpytest()
+    assert_outcomes(result, passed=0, failed=1, error=1, rerun=2)
+    assert "AssertionError: call error" in result.stdout.str()
+    assert "ValueError: teardown error" in result.stdout.str()
+
+
+def test_nonmatching_setup_failure_is_preserved(testdir):
+    testdir.makepyfile(
+        """
+        import pytest
+
+        @pytest.fixture(autouse=True)
+        def teardown_error():
+            yield
+            raise ValueError("teardown error")
+
+        @pytest.fixture
+        def setup_error():
+            raise AssertionError("setup error")
+
+        @pytest.mark.flaky(reruns=1, only_rerun=["ValueError"])
+        def test_fail(setup_error):
+            pass
+        """
+    )
+    result = testdir.runpytest()
+    assert_outcomes(result, passed=0, error=2, rerun=2)
+    assert "AssertionError: setup error" in result.stdout.str()
+    assert "ValueError: teardown error" in result.stdout.str()
+
+
+def test_rerun_except_preserves_terminal_call_failure(testdir):
+    testdir.makepyfile(
+        """
+        import pytest
+
+        @pytest.fixture
+        def teardown_error():
+            yield
+            raise RuntimeError("teardown error")
+
+        @pytest.mark.flaky(reruns=1, rerun_except=["AssertionError"])
+        def test_fail(teardown_error):
+            assert False, "call error"
+        """
+    )
+    result = testdir.runpytest()
+    assert_outcomes(result, passed=0, failed=1, error=1, rerun=0)
+    assert "AssertionError: call error" in result.stdout.str()
+    assert "RuntimeError: teardown error" in result.stdout.str()