diff --git a/README.rst b/README.rst index 747072e..f77c44c 100644 --- a/README.rst +++ b/README.rst @@ -304,6 +304,20 @@ This is useful in large test suites to bound resource usage when many tests are flaky at the same time. The cap applies after rerun selection, including tests configured with ``--force-reruns`` and ``@pytest.mark.flaky``. +Limit reruns when exiting early +------------------------------- + +By default reruns still happen when ``-x``/``--exitfirst`` or ``--maxfail`` +is used, which delays the early exit until a test's reruns are exhausted. +To use a different rerun count in that case, pass ``--reruns-on-exitfirst``: + +.. code-block:: bash + + $ pytest --reruns 2 -x --reruns-on-exitfirst 0 + +The value overrides both the global ``--reruns`` setting and per-test +``flaky`` markers, but not ``--force-reruns``. + Show tracebacks for retried failures ------------------------------------ diff --git a/changes/249.feature.rst b/changes/249.feature.rst new file mode 100644 index 0000000..85b5716 --- /dev/null +++ b/changes/249.feature.rst @@ -0,0 +1,3 @@ +Add ``--reruns-on-exitfirst`` option to use a different rerun count when +``-x``/``--exitfirst`` or ``--maxfail`` is given, e.g. ``--reruns-on-exitfirst +0`` to skip reruns entirely when exiting on the first failure. diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 616e658..9b480bf 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -178,6 +178,15 @@ def pytest_addoption(parser): "'rerun test summary info' section, which is emitted automatically " "when this flag is set.", ) + group._addoption( + "--reruns-on-exitfirst", + action="store", + dest="reruns_on_exitfirst", + type=int, + help="Number of reruns to use when -x/--exitfirst or --maxfail is " + "given. Defaults to the regular rerun settings. Use " + "--reruns-on-exitfirst 0 to skip reruns entirely when exiting early.", + ) group.addoption( "--max-suite-reruns", action="store", @@ -226,7 +235,16 @@ def check_options(config): and config.option.max_suite_reruns < 0 ): raise pytest.UsageError("--max-suite-reruns must be >= 0") + if ( + config.option.reruns_on_exitfirst is not None + and config.option.reruns_on_exitfirst < 0 + ): + raise pytest.UsageError("--reruns-on-exitfirst must be >= 0") reruns = config.getoption("force_reruns") or _get_global_reruns(config) + if not reruns and config.option.maxfail: + # --reruns-on-exitfirst takes effect under -x/--maxfail even when + # no other rerun count is configured + reruns = config.getoption("reruns_on_exitfirst") if not config.getoption("collectonly") and reruns: if config.option.usepdb: # a core option raise pytest.UsageError("--reruns incompatible with --pdb") @@ -250,6 +268,13 @@ def get_reruns_count(item): if reruns is not None: return reruns + if item.session.config.option.maxfail: + reruns = item.session.config.getoption("reruns_on_exitfirst") + if reruns is not None: + # -x/--exitfirst or --maxfail given: override marker and global + # rerun counts so the run can exit on the first real failure. + return reruns + rerun_marker = _get_marker(item) # use the marker as a priority over the global setting. if rerun_marker is not None: diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index a4208c5..19e4257 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -548,6 +548,70 @@ def test_pass(): assert_outcomes(result, passed=1, rerun=2) +def test_reruns_on_exitfirst_zero_disables_reruns(testdir): + testdir.makepyfile("def test_fail(): assert False") + result = testdir.runpytest("--reruns", "2", "-x", "--reruns-on-exitfirst", "0") + assert_outcomes(result, passed=0, failed=1, rerun=0) + + +def test_reruns_on_exitfirst_overrides_flaky_marker(testdir): + testdir.makepyfile( + f""" + import pytest + @pytest.mark.flaky(reruns=2) + def test_fail(): + {temporary_failure()}""" + ) + result = testdir.runpytest("-x", "--reruns-on-exitfirst", "0") + assert_outcomes(result, passed=0, failed=1, rerun=0) + + +def test_reruns_on_exitfirst_limits_reruns(testdir): + testdir.makepyfile( + f""" + def test_fail(): + {temporary_failure(3)}""" + ) + result = testdir.runpytest("--reruns", "3", "-x", "--reruns-on-exitfirst", "1") + assert_outcomes(result, passed=0, failed=1, rerun=1) + + +def test_reruns_on_exitfirst_applies_to_maxfail(testdir): + testdir.makepyfile("def test_fail(): assert False") + result = testdir.runpytest( + "--reruns", "2", "--maxfail", "2", "--reruns-on-exitfirst", "0" + ) + assert_outcomes(result, passed=0, failed=1, rerun=0) + + +def test_exitfirst_without_reruns_on_exitfirst_still_reruns(testdir): + testdir.makepyfile( + f""" + def test_fail(): + {temporary_failure(2)}""" + ) + result = testdir.runpytest("--reruns", "2", "-x") + assert_outcomes(result, passed=1, rerun=2) + + +def test_reruns_on_exitfirst_negative_rejected(testdir): + testdir.makepyfile("def test_pass(): pass") + result = testdir.runpytest("-x", "--reruns-on-exitfirst", "-1") + result.stderr.fnmatch_lines_random("ERROR: --reruns-on-exitfirst must be >= 0") + + +def test_error_when_run_with_pdb_and_reruns_on_exitfirst(testdir): + testdir.makepyfile("def test_pass(): pass") + result = testdir.runpytest("-x", "--reruns-on-exitfirst", "1", "--pdb") + result.stderr.fnmatch_lines_random("ERROR: --reruns incompatible with --pdb") + + +def test_no_error_when_run_with_pdb_and_zero_reruns_on_exitfirst(testdir): + testdir.makepyfile("def test_pass(): pass") + result = testdir.runpytest("-x", "--reruns-on-exitfirst", "0", "--pdb") + assert_outcomes(result) + + def test_no_extra_test_summary_for_reruns_by_default(testdir): testdir.makepyfile( f"""