diff --git a/changes/261.bugfix.rst b/changes/261.bugfix.rst new file mode 100644 index 0000000..71d8ae6 --- /dev/null +++ b/changes/261.bugfix.rst @@ -0,0 +1 @@ +Report a test as rerun when a teardown exception matches the rerun filter after an earlier test-phase failure. diff --git a/changes/385.bugfix.rst b/changes/385.bugfix.rst new file mode 100644 index 0000000..65504e5 --- /dev/null +++ b/changes/385.bugfix.rst @@ -0,0 +1 @@ +Preserve all failed test phases when a later teardown failure triggers a rerun. diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index db3974a..1eb9020 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -731,22 +731,6 @@ def _should_hard_fail_on_error(item, report, excinfo): return (not matches_rerun_only) or matches_rerun_except -def _should_not_rerun(item, report, reruns, condition): - xfail = hasattr(report, "wasxfail") - is_terminal_error = any(item._terminal_errors.values()) - has_failed_subtests = report.when == "call" and _get_num_failed_subtests(item) > 0 - - if ( - item.execution_count > reruns - or (not report.failed and not has_failed_subtests) - or xfail - or is_terminal_error - ): - return True - - return not condition - - def is_master(config): return not (hasattr(config, "workerinput") or hasattr(config, "slaveinput")) @@ -1116,8 +1100,11 @@ def _get_reruns_condition_failures(item): return failures -def _reruns_condition_matches_phase(item, phase): - """Return whether a failed phase matched the attempt's condition.""" +def _rerun_matches_phase(item, phase): + """Return whether a failed phase can trigger a rerun.""" + if item._terminal_errors.get(phase, False): + return False + rerun_marker = _get_marker(item) if rerun_marker is None or "condition" not in rerun_marker.kwargs: return True @@ -1129,6 +1116,36 @@ def _reruns_condition_matches_phase(item, phase): ) +def _has_rerun_except_terminal_error(item): + """Return whether an exception explicitly excluded reruns this attempt.""" + rerun_except_errors = _get_rerun_filter_regex(item, "rerun_except") + if not rerun_except_errors: + return False + + return any( + _matches_any_rerun_except_error(rerun_except_errors, excinfo) + for phase_excinfos in item._rerun_condition_excinfos.values() + for excinfo in phase_excinfos + ) + + +def _should_rerun(item, reruns, condition): + """Return whether this attempt should be rerun as a whole.""" + if ( + not condition + or item.execution_count > reruns + or _has_rerun_except_terminal_error(item) + or item._test_xfailed.get("setup", False) + or item._test_xfailed.get("call", False) + ): + return False + + return any( + _rerun_matches_phase(item, phase) + for phase, _, _ in _get_reruns_condition_failures(item) + ) + + def _teardown_suspended_finalizers(item, call, report): """Tear down the scopes held back for a re-run that will not happen. @@ -1303,56 +1320,51 @@ def pytest_runtest_protocol(item, nextitem): item.ihook.pytest_runtest_logstart(nodeid=item.nodeid, location=item.location) reports = runtestprotocol(item, nextitem=nextitem, log=False) - condition = get_reruns_condition(item, _get_reruns_condition_failures(item)) - rerun_triggered = False + condition_failures = _get_reruns_condition_failures(item) + condition = get_reruns_condition(item, condition_failures) + rerun_triggered = _should_rerun(item, reruns, condition) + + if rerun_triggered: + max_suite_reruns = item.session.config.option.max_suite_reruns + if max_suite_reruns is not None and not db.try_increment_suite_reruns( + max_suite_reruns + ): + # Suite-wide limit exhausted -- log all reports as final results. + _restore_suspended_finalizers(item) + rerun_triggered = False + + rerun_report = next( + ( + report + for report in reports + if report.when == "call" and _get_num_failed_subtests(item) > 0 + ), + next((report for report in reports if report.failed), None), + ) for report in reports: # 3 reports: setup, call, teardown report.rerun = item.execution_count - 1 - if rerun_triggered: - if report.failed: - report.outcome = "rerun" - item.ihook.pytest_runtest_logreport(report=report) - elif ( - condition - and not _reruns_condition_matches_phase(item, report.when) - and ( - report.failed - or (report.when == "call" and _get_num_failed_subtests(item) > 0) - ) + if rerun_triggered and ( + report.failed + or (report.when == "call" and _get_num_failed_subtests(item) > 0) ): - # Another failed phase matched the condition and will carry - # this intermediate attempt's rerun report. Do not publish a - # nonmatching failure as a final result first. - continue - elif _should_not_rerun(item, report, reruns, condition): - # no rerun needed or one already triggered, log normally - item.ihook.pytest_runtest_logreport(report=report) - else: - # failure detected and reruns not exhausted, since i < reruns - max_suite_reruns = item.session.config.option.max_suite_reruns - if max_suite_reruns is not None: - if not db.try_increment_suite_reruns(max_suite_reruns): - # Suite-wide limit exhausted -- log as final failure. - _restore_suspended_finalizers(item) - item.ihook.pytest_runtest_logreport(report=report) - continue - report.outcome = "rerun" - time.sleep(delay * delay_backoff_factor ** (item.execution_count - 1)) - if not parallel or works_with_current_xdist(): - # will rerun test, log intermediate result item.ihook.pytest_runtest_logreport(report=report) + else: + item.ihook.pytest_runtest_logreport(report=report) - # cleanin item's cashed results from any level of setups - _remove_cached_results_from_failed_fixtures(item) - _remove_failed_setup_state_from_session(item) - _discard_test_class_instance(item) - _remove_failed_subtests_from_report(item, report) - _remove_failed_subtest_reports_from_stats( - item.config, item.session, item.nodeid - ) + if rerun_triggered: + time.sleep(delay * delay_backoff_factor ** (item.execution_count - 1)) - rerun_triggered = True + # Clean cached results from any level of setups. + _remove_cached_results_from_failed_fixtures(item) + _remove_failed_setup_state_from_session(item) + _discard_test_class_instance(item) + if rerun_report is not None: + _remove_failed_subtests_from_report(item, rerun_report) + _remove_failed_subtest_reports_from_stats( + item.config, item.session, item.nodeid + ) # Do not retain ExceptionInfo tracebacks and their frame locals for the # lifetime of the collected item/session. diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index 2094448..2ab6ef1 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -2313,7 +2313,7 @@ def test_service(service): result = testdir.runpytest() assert result.ret == 0 - assert_outcomes(result, passed=1, rerun=1) + assert_outcomes(result, passed=1, rerun=2) def test_condition_exception_state_is_released_after_attempt(testdir): @@ -2616,6 +2616,33 @@ def test_fail(broken_fixture): assert result.stdout.str().count("module teardown") == 1 +def test_teardown_error_can_trigger_only_rerun(testdir): + testdir.makepyfile( + """ + import pytest + + attempts = 0 + + @pytest.fixture + def broken_fixture(): + yield + if attempts == 1: + raise ValueError("teardown error") + + @pytest.mark.flaky(reruns=1, only_rerun=["ValueError"]) + def test_fail(broken_fixture): + global attempts + attempts += 1 + if attempts == 1: + raise AssertionError("call error") + """ + ) + + result = testdir.runpytest() + + assert_outcomes(result, passed=1, rerun=2) + + @pytest.mark.parametrize( "outcome,skipped,xfailed", [("skip", 3, 0), ("xfail", 0, 3)], @@ -3890,3 +3917,70 @@ def test_fail(): ) result = testdir.runpytest("-s", "--reruns", "1", "--max-suite-reruns", "0") result.stdout.fnmatch_lines("*module teardown*") + + +def test_nonmatching_call_failure_is_preserved_on_last_rerun(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.fixture + def teardown_error(): + yield + raise ValueError("teardown error") + + @pytest.mark.flaky(reruns=1, only_rerun=["ValueError"]) + def test_fail(teardown_error): + raise AssertionError("call error") + """ + ) + result = testdir.runpytest() + assert_outcomes(result, passed=0, failed=1, error=1, rerun=2) + assert "AssertionError: call error" in result.stdout.str() + assert "ValueError: teardown error" in result.stdout.str() + + +def test_nonmatching_setup_failure_is_preserved(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.fixture(autouse=True) + def teardown_error(): + yield + raise ValueError("teardown error") + + @pytest.fixture + def setup_error(): + raise AssertionError("setup error") + + @pytest.mark.flaky(reruns=1, only_rerun=["ValueError"]) + def test_fail(setup_error): + pass + """ + ) + result = testdir.runpytest() + assert_outcomes(result, passed=0, error=2, rerun=2) + assert "AssertionError: setup error" in result.stdout.str() + assert "ValueError: teardown error" in result.stdout.str() + + +def test_rerun_except_preserves_terminal_call_failure(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.fixture + def teardown_error(): + yield + raise RuntimeError("teardown error") + + @pytest.mark.flaky(reruns=1, rerun_except=["AssertionError"]) + def test_fail(teardown_error): + assert False, "call error" + """ + ) + result = testdir.runpytest() + assert_outcomes(result, passed=0, failed=1, error=1, rerun=0) + assert "AssertionError: call error" in result.stdout.str() + assert "RuntimeError: teardown error" in result.stdout.str()