From a1c4ca9901b5ad317ff8cbfe1af0cfcd8f608e96 Mon Sep 17 00:00:00 2001 From: Akim Akimov Date: Thu, 19 Nov 2026 04:52:36 +0000 Subject: [PATCH] ci: prewarm the assertion-rewrite cache before backend shards Every xdist worker of a backend shard collects the whole shard and, on a fresh runner, rewrites the asserts of each test file because no cached pyc exists. A new step before the full-suite test step writes those pycs once, one process per core, with pytest's own rewrite functions, under the cache root the rootdir conftest sets. It parses and compiles source and imports no test module, conftest or kiro_crew, unlike the serial --collect-only prewarm that measured slower. The step is best effort: it exits 0 whatever happens and pytest rewrites any file it did not cache. It does not choose what a shard collects. This is an experiment with a ship gate, measured on the pull request's own full-suite run (#39368). --- .github/workflows/ci.yml | 39 ++- docs/ci/ci-and-reviews.md | 16 + .../common/testing-conventions.md | 9 + scripts/ci_rewrite_prewarm.py | 308 +++++++++++++++++ test/test_ci_rewrite_prewarm.py | 323 ++++++++++++++++++ 5 files changed, 692 insertions(+), 3 deletions(-) create mode 100644 scripts/ci_rewrite_prewarm.py create mode 100644 test/test_ci_rewrite_prewarm.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d6788c5b243..796fee5fb8d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1361,6 +1361,25 @@ jobs: echo "reduced=true" >> "$GITHUB_OUTPUT" echo "Reduced scope: $(wc -l < "$TARGETS_FILE") cross-surface backend file(s)." + # Experiment (#39368): each xdist worker collects the whole shard and + # rewrites the asserts of every test file, because a fresh runner has no + # cached pyc. This writes those pycs once, one process per core, without + # importing a test module, conftest or kiro_crew, into the cache root the + # rootdir conftest points pytest at. Full suite only: it is the arm that + # takes --file-shard-pins. Same shell as the test step, so the same user + # and home, which is what the cache root is derived from. Best effort: + # the script exits 0 whatever happens, and a file it did not cache is + # rewritten by pytest. It prints its own elapsed seconds. + - name: Prewarm assertion rewrites (shard ${{ matrix.group }}/${{ env.SHARD_COUNT }}) + if: ${{ steps.scope.outputs.reduced == 'false' }} + shell: /usr/local/bin/ci-shell {0} + continue-on-error: true + timeout-minutes: 5 + run: | + python -m scripts.ci_rewrite_prewarm \ + --file-shards "$SHARD_COUNT" --file-shard ${{ matrix.group }} \ + --file-shard-pins scripts/ci_file_shard_pins.txt + - name: Run tests (shard ${{ matrix.group }}/${{ env.SHARD_COUNT }}) shell: /usr/local/bin/ci-shell {0} # scripts.ci_file_shards assigns each file to one shard before pytest @@ -1639,9 +1658,11 @@ jobs: # full (main pushes 44376034204, 44365593626): 26.6 min wall, 183-191 job-min # reduced, 1,342 files (run 44426610212): 8.7 min wall, 60 job-min # leaf, 1 file + 909 gates (run 44427119854): 13.8 min wall, 64 job-min - # No pytest prewarm: a serial `--collect-only` of one shard costs 104-115 s - # and saves 42-75 s of worker collection, so it loses on both OSes, with - # and without --cov (prewarm probe in run 44426610212). + # No `--collect-only` prewarm: a serial one costs 104-115 s per shard and + # saves 42-75 s of worker collection, so it loses on both OSes, with and + # without --cov (prewarm probe in run 44426610212). The step before + # "Run tests" is a different form: it only rewrites asserts, in parallel, + # and imports nothing. - name: Select test scope id: scope shell: bash @@ -1713,6 +1734,18 @@ jobs: echo "reduced=true" >> "$GITHUB_OUTPUT" echo "Reduced scope: $(wc -l < "$TARGETS_FILE") cross-surface backend file(s)." + # Same experiment and contract as backend-test's prewarm step (#39368), + # with the Windows pin file and the test step's shell. + - name: Prewarm assertion rewrites (Windows shard ${{ matrix.group }}/${{ env.SHARD_COUNT }}) + if: ${{ steps.scope.outputs.reduced == 'false' }} + shell: bash + continue-on-error: true + timeout-minutes: 5 + run: | + python -m scripts.ci_rewrite_prewarm \ + --file-shards "$SHARD_COUNT" --file-shard ${{ matrix.group }} \ + --file-shard-pins scripts/ci_file_shard_pins_windows.txt + - name: Run tests (Windows shard ${{ matrix.group }}/${{ env.SHARD_COUNT }}) # bash (Git Bash on windows-latest), not pwsh: the invocation shares # the POSIX job's line-continuation and env-interpolation syntax. diff --git a/docs/ci/ci-and-reviews.md b/docs/ci/ci-and-reviews.md index e51386a4afd1..52ff333199bc 100644 --- a/docs/ci/ci-and-reviews.md +++ b/docs/ci/ci-and-reviews.md @@ -884,6 +884,22 @@ coverage or a repair of the crashing test. Periodic console records and best-eff JSONL artifacts preserve evidence; job-level cancellation can prevent uploads. Healthy selection, per-test timeouts and coverage selectors are unchanged. +On the full suite, the step before `Run tests` in both jobs runs +`scripts/ci_rewrite_prewarm.py` (an experiment, #39368). Every xdist worker +collects the whole shard and, on a fresh runner, rewrites the asserts of each +test file because no cached pyc exists. The step writes those pycs once, one +process per core, with pytest's own rewrite functions, under the cache root the +rootdir conftest sets (`sys.pycache_prefix`), for the test files the shard owns +and every `conftest.py`. It parses and compiles source and imports no test +module, conftest or `kiro_crew`, which is what separates it from the serial +`--collect-only` prewarm that measured slower. It does not choose what a shard +collects: pytest's discovery and the plugin still do. It is best effort: it +exits 0 whatever happens, a file it did not cache is rewritten by pytest, and +the reduced and leaf scopes skip it. Bytecode is never restored from +`actions/cache` or committed. The step prints its file count and elapsed +seconds. `test/test_ci_rewrite_prewarm.py` proves pytest reads the prewarmed pyc +and pins the cache root against the conftest's. + The root conftest's import-time telemetry guard fails the offending module's collection report on every worker, so pytest/xdist fails the job even when the shard does not own `test_host_isolation_floor.py`. The process-wide telemetry-off diff --git a/docs/system-specs/common/testing-conventions.md b/docs/system-specs/common/testing-conventions.md index 3785c97a1d67..1b433ca50876 100644 --- a/docs/system-specs/common/testing-conventions.md +++ b/docs/system-specs/common/testing-conventions.md @@ -6993,6 +6993,15 @@ Linux pins cut the slowest Windows shard by a median 12% of its summed test seconds over six held-out green runs, the Windows pins by 20%. macOS, release and the reduced and leaf scopes keep pure hash ownership. +Before the Linux and Windows full-suite test step, `scripts/ci_rewrite_prewarm.py` +writes pytest's assertion-rewrite pyc for the shard's test files (same hash and +pin file) and every `conftest.py`, in parallel and without importing them, under +the `sys.pycache_prefix` the rootdir conftest sets. It changes where a worker's +collection time goes, not which files the shard collects, and never fails the +job. `test/test_ci_rewrite_prewarm.py` owns the proof that pytest reads those +pycs; a change to the cache root in `test/conftest_floor_configure.py` must keep +`ci_rewrite_prewarm.pycache_prefix` equal to it. + macOS assigns whole files like the other platforms, in eight shards. The shard count is the knob to reach for when the suite grows, not the 60-minute cap, because that cap is a spend guard as much as a hang guard on a lane billed at ten diff --git a/scripts/ci_rewrite_prewarm.py b/scripts/ci_rewrite_prewarm.py new file mode 100644 index 000000000000..986d99c882f6 --- /dev/null +++ b/scripts/ci_rewrite_prewarm.py @@ -0,0 +1,308 @@ +"""Write pytest's assertion-rewrite cache for one CI file shard, in parallel. + +Every xdist worker of a shard collects the whole shard, and on a fresh runner +each of them rewrites the asserts of every test file because no cached pyc +exists. This script produces those pycs once, across one process per core, +before pytest starts, so each worker reads a cached pyc instead. + +It parses and compiles source text and never executes it: no test module, +``conftest.py`` or ``kiro_crew`` import happens here. The cache format, file +name and directory are pytest's own (``_rewrite_test``, ``_write_pyc``, +``PYC_TAIL`` and ``get_cache_dir`` from ``_pytest.assertion.rewrite``), so the +pyc always matches the installed pytest. + +What is rewritten: + +- the test files this shard owns: every file under ``testpaths`` whose name + matches ``python_files``, kept when ``scripts.ci_file_shards.file_shard`` + names this shard as its owner (same hash, same pin file as the test step); +- every ``conftest.py`` under the rootdir and ``testpaths``, which each worker + of every shard loads. + +What is not: modules pytest rewrites only because something registered them at +import time (``pytest_plugins``, ``register_assert_rewrite``), and files a +conftest ``pytest_ignore_collect`` hook would drop. The first group is rewritten +by pytest as before; the second gets a pyc nobody reads. Neither changes which +files a shard collects: pytest's own discovery still decides that. + +Best effort by contract: a file that cannot be rewritten is reported and left to +pytest, a run that exceeds its budget stops, and the exit status is 0 in every +case. + +Run from the checkout root, as the user and in the directory the test step uses: +``python -m scripts.ci_rewrite_prewarm --file-shards N --file-shard K``. +""" + +from __future__ import annotations + +import argparse +import configparser +import fnmatch +import multiprocessing +import os +import sys +import time +from collections.abc import Iterable, Mapping +from pathlib import Path + +from scripts.ci_file_shards import file_shard, read_pins + +_DEFAULT_PYTHON_FILES = ("test_*.py", "*_test.py") +# pytest's own ``norecursedirs`` default, plus the bytecode directory. +_DEFAULT_NORECURSEDIRS = ( + "*.egg", + ".*", + "_darcs", + "build", + "CVS", + "dist", + "node_modules", + "venv", + "{arch}", +) +_BUDGET_SECONDS = 240.0 + + +def pycache_prefix() -> str | None: + """The directory the suite's rootdir conftest sends bytecode to, or None. + + Mirrors ``_redirect_bytecode_cache`` in ``test/conftest_floor_configure.py``: + ``$XDG_CACHE_HOME``, else ``.cache`` under the account's real home, then + ``kirocrew/pycache``; nothing when bytecode writing is off. + ``test/test_ci_rewrite_prewarm.py`` pins the two against each other. + """ + if os.environ.get("PYTHONDONTWRITEBYTECODE"): + return None + cache_root = os.environ.get("XDG_CACHE_HOME") + if not cache_root: + home = None + if os.name != "nt": + try: + import pwd + + home = pwd.getpwuid(os.getuid()).pw_dir + except (ImportError, KeyError, OSError): + home = None + cache_root = os.path.join(home or str(Path.home()), ".cache") + return os.path.join(cache_root, "kirocrew", "pycache") + + +def _pytest_ini(root: Path) -> Mapping[str, str]: + """The ``[tool:pytest]`` section of the rootdir ``setup.cfg``, or nothing.""" + parser = configparser.ConfigParser(interpolation=None) + try: + parser.read(root / "setup.cfg", encoding="utf-8") + except (configparser.Error, OSError): + return {} + return dict(parser["tool:pytest"]) if parser.has_section("tool:pytest") else {} + + +def _words(value: str | None, default: Iterable[str]) -> tuple[str, ...]: + """Split an ini list value on whitespace, dropping ``#`` comment lines.""" + if value is None: + return tuple(default) + lines = (line for line in value.splitlines() if not line.lstrip().startswith("#")) + return tuple(word for line in lines for word in line.split()) + + +def assertion_pass_hook_enabled(root: Path) -> bool: + """Whether the ini turns on a rewrite variant this script does not produce.""" + value = _pytest_ini(root).get("enable_assertion_pass_hook", "") + return value.strip().lower() in ("1", "true", "yes", "on") + + +def discover(root: Path) -> tuple[list[Path], list[Path]]: + """Return (test files, conftest files) under the rootdir's ``testpaths``. + + Reads file names only. A superset of what pytest collects is fine (a + conftest ignore hook is not consulted); a subset only leaves work to pytest. + """ + ini = _pytest_ini(root) + patterns = _words(ini.get("python_files"), _DEFAULT_PYTHON_FILES) + skip = (*_words(ini.get("norecursedirs"), _DEFAULT_NORECURSEDIRS), "__pycache__") + tests: list[Path] = [] + conftests: list[Path] = [] + if (root / "conftest.py").is_file(): + conftests.append(root / "conftest.py") + for testpath in _words(ini.get("testpaths"), (".",)): + for directory, subdirs, names in os.walk(root / testpath): + subdirs[:] = sorted( + name + for name in subdirs + if not any(fnmatch.fnmatch(name, pattern) for pattern in skip) + and not os.path.isfile(os.path.join(directory, name, "pyvenv.cfg")) + ) + for name in sorted(names): + path = Path(directory, name) + if name == "conftest.py": + if path not in conftests: + conftests.append(path) + elif any(fnmatch.fnmatch(name, pattern) for pattern in patterns): + tests.append(path) + return tests, conftests + + +def shard_files( + root: Path, count: int, index: int, pins: Mapping[str, int] | None = None +) -> list[Path]: + """The files to rewrite for one shard: its own test files, then every conftest.""" + tests, conftests = discover(root) + owned = [path for path in tests if file_shard(path, root, count, pins) == index] + return owned + conftests + + +class _Trace: + """Stands in for pytest's ``AssertionState``: ``_write_pyc`` only calls ``trace``.""" + + def __init__(self) -> None: + self.messages: list[str] = [] + + def trace(self, message: str) -> None: + self.messages.append(message) + + +def _use_prefix(prefix: str | None) -> None: + """Point this process at the cache root pytest will read (pool initializer).""" + sys.pycache_prefix = prefix + + +def rewrite_file(source: str) -> str | None: + """Rewrite one file's asserts and cache the result; return why not, or None. + + Never raises: a syntax error, an unreadable file or an unwritable cache + directory all leave the file for pytest to rewrite itself. + """ + try: + from _pytest.assertion import rewrite + + path = Path(source) + cache_dir = rewrite.get_cache_dir(path) + if not rewrite.try_makedirs(cache_dir): + return f"cannot create {cache_dir}" + # No Config: rewrite_asserts takes None and then rewrites without the + # assertion-pass hook, which is the variant the ini selects (checked in _run). + source_stat, code = rewrite._rewrite_test(path, None) # type: ignore[arg-type] + state = _Trace() + pyc = cache_dir / (path.name[:-3] + rewrite.PYC_TAIL) + if not rewrite._write_pyc(state, code, source_stat, pyc): # type: ignore[arg-type] + return "; ".join(state.messages) or f"cannot write {pyc}" + except Exception as exc: # noqa: BLE001 -- best effort: pytest rewrites this file + return f"{type(exc).__name__}: {exc}" + return None + + +def _rewrite_reporting(source: str) -> tuple[str, str | None]: + return source, rewrite_file(source) + + +def prewarm( + files: list[Path], prefix: str | None, jobs: int, budget: float = _BUDGET_SECONDS +) -> tuple[int, list[tuple[str, str]], bool]: + """Rewrite ``files`` across ``jobs`` processes. + + Returns (files cached, [(file, reason)] skipped, whether the budget ran out). + Largest files go first so the processes finish close together. ``spawn`` on + every OS: the children start from a clean interpreter, as on Windows. + """ + ordered = sorted(files, key=lambda path: (-_size(path), str(path))) + sources = [str(path) for path in ordered] + written = 0 + skipped: list[tuple[str, str]] = [] + if not sources: + return written, skipped, False + deadline = time.monotonic() + budget + context = multiprocessing.get_context("spawn") + pool = context.Pool(max(1, min(jobs, len(sources))), _use_prefix, (prefix,)) + try: + # One file per task: only the unchunked iterator takes a timeout. + results = pool.imap_unordered(_rewrite_reporting, sources) + for _ in sources: + try: + source, reason = results.next(timeout=max(0.0, deadline - time.monotonic())) + except multiprocessing.TimeoutError: + return written, skipped, True + if reason is None: + written += 1 + else: + skipped.append((source, reason)) + finally: + pool.terminate() + pool.join() + return written, skipped, False + + +def _size(path: Path) -> int: + try: + return path.stat().st_size + except OSError: + return 0 + + +def _cores() -> int: + try: + return len(os.sched_getaffinity(0)) # type: ignore[attr-defined,unused-ignore] + except (AttributeError, OSError): + return os.cpu_count() or 1 + + +def _run(argv: list[str] | None) -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--file-shards", type=int, required=True) + parser.add_argument("--file-shard", type=int, required=True) + parser.add_argument("--file-shard-pins", default=None) + parser.add_argument("--jobs", type=int, default=None, help="processes; default: one per core") + parser.add_argument( + "--pycache-prefix", + default=None, + help="cache root; default: the one the rootdir conftest sets", + ) + parser.add_argument("--budget-seconds", type=float, default=_BUDGET_SECONDS) + args = parser.parse_args(argv) + started = time.perf_counter() + label = f"rewrite prewarm (shard {args.file_shard}/{args.file_shards})" + # The rootdir is where pytest is started, spelled as the OS reports it: + # pytest builds module paths from the same string, and the cache + # directory is derived from that path. + root = Path(os.getcwd()) + if not 1 <= args.file_shard <= args.file_shards: + print(f"{label}: skipped, shard index out of range") + return + if args.pycache_prefix is None and os.environ.get("PYTHONDONTWRITEBYTECODE"): + print(f"{label}: skipped, PYTHONDONTWRITEBYTECODE is set so pytest reads no cache") + return + if assertion_pass_hook_enabled(root): + print(f"{label}: skipped, enable_assertion_pass_hook changes the rewrite") + return + prefix = args.pycache_prefix or pycache_prefix() + pins = read_pins(Path(args.file_shard_pins), args.file_shards) if args.file_shard_pins else {} + files = shard_files(root, args.file_shards, args.file_shard, pins) + jobs = args.jobs or _cores() + written, skipped, timed_out = prewarm(files, prefix, jobs, args.budget_seconds) + for source, reason in skipped: + print(f"{label}: left to pytest: {source}: {reason}") + if timed_out: + print(f"{label}: stopped at the {args.budget_seconds:.0f} s budget; pytest does the rest") + elapsed = time.perf_counter() - started + print( + f"{label}: {written} of {len(files)} files cached, {len(skipped)} left to pytest, " + f"{jobs} processes, elapsed {elapsed:.1f} s, cache {prefix}" + ) + + +def main(argv: list[str] | None = None) -> int: + """Always 0: this step saves time or does nothing, it never fails a job.""" + try: + # A path the console encoding cannot spell is printed escaped. + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(errors="backslashreplace") + _run(argv) + except SystemExit as exc: # argparse usage error + print(f"rewrite prewarm: skipped, bad arguments (exit {exc.code})") + except Exception as exc: # noqa: BLE001 -- best effort by contract + print(f"rewrite prewarm: skipped, {type(exc).__name__}: {exc}") + sys.stdout.flush() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/test_ci_rewrite_prewarm.py b/test/test_ci_rewrite_prewarm.py new file mode 100644 index 000000000000..cd357010d685 --- /dev/null +++ b/test/test_ci_rewrite_prewarm.py @@ -0,0 +1,323 @@ +"""The CI rewrite prewarm writes the pyc pytest reads, and imports nothing.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml +from conftest_floor_configure import _redirect_bytecode_cache, _shared_cache_root + +from kiro_crew.testing.wait import FALLBACK_TIMEOUT_SECS +from scripts import ci_rewrite_prewarm +from scripts.ci_file_shards import file_shard + +ROOT = Path(__file__).resolve().parents[1] +# Three bounded children at most per test, so an expired child fails by name +# before pytest-timeout would kill the xdist worker (Windows has no SIGALRM). +pytestmark = pytest.mark.timeout(4 * FALLBACK_TIMEOUT_SECS) + +_IMPORT_MARKER = "open(__file__ + '.imported', 'w', encoding='utf-8').close()\n" + + +def _write(root: Path, name: str, content: str) -> Path: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +def _tree(tmp_path: Path) -> Path: + """A checkout-shaped tree, spelled as the OS reports the working directory.""" + root = Path(os.path.realpath(tmp_path)) / "checkout" + _write(root, "setup.cfg", "[tool:pytest]\ntestpaths = test apps\n") + _write(root, "conftest.py", "ROOT_CONFTEST = True\n") + _write(root, "test/conftest.py", "TEST_CONFTEST = True\n") + _write(root, "test/test_alpha.py", "def test_alpha():\n assert 1 + 1 == 2\n") + _write(root, "test/nested/test_beta.py", "def test_beta():\n assert 'b' in 'abc'\n") + _write(root, "apps/demo/tests/test_gamma.py", "def test_gamma():\n assert [1] == [1]\n") + return root + + +def _child(root: Path, prefix: Path, *args: str) -> subprocess.CompletedProcess[str]: + """Run ``python *args`` in ``root`` with bytecode redirected to ``prefix``.""" + env = dict(os.environ) + for name in ("PYTEST_ADDOPTS", "PYTEST_PLUGINS", "PYTHONDONTWRITEBYTECODE"): + env.pop(name, None) + for name in ("COV_CORE_SOURCE", "COV_CORE_DATAFILE", "COVERAGE_PROCESS_START"): + env.pop(name, None) + env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + env["PYTHONPATH"] = str(ROOT) + env["PYTHONPYCACHEPREFIX"] = str(prefix) + env["TMPDIR"] = env["TMP"] = env["TEMP"] = str(root.parent) + try: + return subprocess.run( + [sys.executable, *args], + cwd=root, + env=env, + capture_output=True, + text=True, + encoding="utf-8", + timeout=FALLBACK_TIMEOUT_SECS, + ) + except subprocess.TimeoutExpired as exc: + pytest.fail(f"{args} still running at its {exc.timeout}s ceiling: {exc.stdout!r}") + + +def _prewarm(root: Path, prefix: Path, *extra: str) -> subprocess.CompletedProcess[str]: + """Run the prewarm for the only shard, caching under ``prefix``. + + The child's own ``PYTHONPYCACHEPREFIX`` points elsewhere, so a pyc under + ``prefix`` got there through the cache root the script was given. + """ + return _child( + root, + root.parent / "decoy-cache", + "-m", + "scripts.ci_rewrite_prewarm", + "--file-shards", + "1", + "--file-shard", + "1", + "--jobs", + "2", + "--pycache-prefix", + str(prefix), + *extra, + ) + + +def _collect_traced(root: Path, prefix: Path) -> tuple[set[str], set[str]]: + """Run pytest on the tree; return the file names it (read from cache, rewrote). + + pytest's assertion rewriter traces one of two lines per test module it + loads, and ``--debug`` writes that trace to a file. + """ + log = root.parent / f"trace-{prefix.name}.log" + result = _child(root, prefix, "-m", "pytest", "-q", "-p", "no:cacheprovider", f"--debug={log}") + assert result.returncode == 0, result.stdout + result.stderr + assert "3 passed" in result.stdout, result.stdout + cached: set[str] = set() + rewritten: set[str] = set() + for line in log.read_text(encoding="utf-8").splitlines(): + for name in ("test_alpha.py", "test_beta.py", "test_gamma.py"): + if name in line and "found cached rewritten pyc for" in line: + cached.add(name) + if name in line and "rewriting " in line: + rewritten.add(name) + return cached, rewritten + + +def _pycs(prefix: Path) -> dict[str, tuple[int, bytes]]: + return { + path.name: (path.stat().st_mtime_ns, path.read_bytes()) for path in prefix.rglob("*.pyc") + } + + +def test_pytest_reads_the_prewarmed_pyc_instead_of_rewriting(tmp_path): + root = _tree(tmp_path) + prefix = root.parent / "cache" + result = _prewarm(root, prefix) + assert result.returncode == 0, result.stdout + result.stderr + assert "5 of 5 files cached, 0 left to pytest" in result.stdout, result.stdout + before = _pycs(prefix) + stems = {name.split(".", 1)[0] for name in before} + assert stems == {"conftest", "test_alpha", "test_beta", "test_gamma"} + assert not list((root.parent / "decoy-cache").rglob("*-pytest-*.pyc")) + + cached, rewritten = _collect_traced(root, prefix) + + assert cached == {"test_alpha.py", "test_beta.py", "test_gamma.py"} + assert rewritten == set() + # pytest replaces a pyc it does not accept, conftest ones included. + after = _pycs(prefix) + assert {name: after[name] for name in before} == before + + +def test_without_the_prewarm_pytest_rewrites_every_file(tmp_path): + root = _tree(tmp_path) + + cached, rewritten = _collect_traced(root, root.parent / "cache") + + assert cached == set() + assert rewritten == {"test_alpha.py", "test_beta.py", "test_gamma.py"} + + +def test_a_pyc_under_another_cache_root_is_not_read(tmp_path): + root = _tree(tmp_path) + assert _prewarm(root, root.parent / "elsewhere").returncode == 0 + + cached, rewritten = _collect_traced(root, root.parent / "cache") + + assert cached == set() + assert rewritten == {"test_alpha.py", "test_beta.py", "test_gamma.py"} + + +def test_prewarm_executes_no_test_module_and_no_conftest(tmp_path): + root = _tree(tmp_path) + for name in ("conftest.py", "test/conftest.py", "test/test_alpha.py"): + _write(root, name, _IMPORT_MARKER + "def test_alpha():\n assert True\n") + prefix = root.parent / "cache" + probe = ( + "import sys\n" + "from scripts import ci_rewrite_prewarm\n" + "code = ci_rewrite_prewarm.main(sys.argv[1:])\n" + "loaded = sorted(m for m in sys.modules if m.split('.')[0] in " + "('kiro_crew', 'conftest', 'test_alpha'))\n" + "print('LOADED', loaded, 'EXIT', code)\n" + ) + args = ("--file-shards", "1", "--file-shard", "1", "--jobs", "2") + result = _child( + root, root.parent / "decoy-cache", "-c", probe, *args, "--pycache-prefix", str(prefix) + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "5 of 5 files cached" in result.stdout, result.stdout + assert "LOADED [] EXIT 0" in result.stdout, result.stdout + assert not list(root.rglob("*.imported")) + # Positive control: pytest does execute them, so the marker would be seen. + assert _child(root, prefix, "-m", "pytest", "-q", "-p", "no:cacheprovider").returncode == 0 + assert {path.name for path in root.rglob("*.imported")} == { + "conftest.py.imported", + "test_alpha.py.imported", + } + + +def test_a_file_that_cannot_be_rewritten_is_left_to_pytest(tmp_path): + root = _tree(tmp_path) + broken = _write(root, "test/test_broken.py", "def test_broken(:\n") + prefix = root.parent / "cache" + + result = _prewarm(root, prefix) + + assert result.returncode == 0, result.stdout + result.stderr + assert "5 of 6 files cached, 1 left to pytest" in result.stdout, result.stdout + assert f"left to pytest: {broken}: SyntaxError" in result.stdout, result.stdout + stems = {name.split(".", 1)[0] for name in _pycs(prefix)} + assert stems == {"conftest", "test_alpha", "test_beta", "test_gamma"} + + +def test_unusable_arguments_never_fail_the_step(tmp_path, capsys): + missing = tmp_path / "no-such-pins.txt" + for argv in ( + ["--file-shards", "many", "--file-shard", "1"], + ["--file-shards", "8", "--file-shard", "9"], + ["--file-shards", "8", "--file-shard", "1", "--file-shard-pins", str(missing)], + ): + assert ci_rewrite_prewarm.main(argv) == 0 + assert "skipped" in capsys.readouterr().out + + +def test_each_shard_rewrites_the_files_the_test_step_assigns_it(tmp_path): + root = _tree(tmp_path) + for index in range(40): + _write(root, f"test/test_file_{index}.py", "def test_ok():\n assert True\n") + tests, conftests = ci_rewrite_prewarm.discover(root) + assert len(tests) == 43 + assert conftests == [root / "conftest.py", root / "test" / "conftest.py"] + pinned = "test/test_alpha.py" + pins = {pinned: 1 + file_shard(root / pinned, root, 4) % 4} + + owned = { + index: ci_rewrite_prewarm.shard_files(root, 4, index, pins)[: -len(conftests)] + for index in range(1, 5) + } + + assert sorted(path for paths in owned.values() for path in paths) == sorted(tests) + for index, paths in owned.items(): + assert paths, f"shard {index} owns nothing" + assert all(file_shard(path, root, 4, pins) == index for path in paths) + assert ci_rewrite_prewarm.shard_files(root, 4, index, pins)[-len(conftests) :] == conftests + assert root / pinned in owned[pins[pinned]] + + +def test_discovery_follows_the_ini_and_skips_what_pytest_never_enters(tmp_path): + root = tmp_path / "checkout" + _write( + root, + "setup.cfg", + "[tool:pytest]\npython_files = check_*.py\ntestpaths =\n # a comment line\n suite\n", + ) + kept = _write(root, "suite/deep/check_kept.py", "") + for name in ( + "suite/test_other_pattern.py", + "suite/.hidden/check_hidden.py", + "suite/node_modules/check_vendored.py", + "suite/__pycache__/check_cached.py", + "suite/env/check_in_virtualenv.py", + "elsewhere/check_outside_testpaths.py", + ): + _write(root, name, "") + _write(root, "suite/env/pyvenv.cfg", "") + + assert ci_rewrite_prewarm.discover(root) == ([kept], []) + assert not ci_rewrite_prewarm.assertion_pass_hook_enabled(root) + _write(root, "setup.cfg", "[tool:pytest]\nenable_assertion_pass_hook = true\n") + assert ci_rewrite_prewarm.assertion_pass_hook_enabled(root) + + +def test_the_repository_ini_keeps_the_plain_rewrite(): + assert not ci_rewrite_prewarm.assertion_pass_hook_enabled(ROOT) + tests, conftests = ci_rewrite_prewarm.discover(ROOT) + assert ROOT / "test" / "test_ci_rewrite_prewarm.py" in tests + assert ROOT / "conftest.py" in conftests + + +def test_cache_root_is_the_one_the_rootdir_conftest_sets(tmp_path, monkeypatch): + monkeypatch.delenv("PYTHONDONTWRITEBYTECODE", raising=False) + monkeypatch.setattr(sys, "pycache_prefix", sys.pycache_prefix) + monkeypatch.setenv("PYTHONPYCACHEPREFIX", os.environ.get("PYTHONPYCACHEPREFIX", "")) + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + + _redirect_bytecode_cache() + + assert sys.pycache_prefix == os.path.join(str(tmp_path), "kirocrew", "pycache") + assert ci_rewrite_prewarm.pycache_prefix() == sys.pycache_prefix + # Without XDG_CACHE_HOME both fall back to the account's real home. + monkeypatch.delenv("XDG_CACHE_HOME") + assert ci_rewrite_prewarm.pycache_prefix() == os.path.join( + _shared_cache_root(), "kirocrew", "pycache" + ) + monkeypatch.setenv("PYTHONDONTWRITEBYTECODE", "1") + assert ci_rewrite_prewarm.pycache_prefix() is None + + +def test_workflow_prewarms_only_the_full_suite_right_before_the_tests(): + workflow = yaml.safe_load((ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8")) + for name, pins in ( + ("backend-test", "scripts/ci_file_shard_pins.txt"), + ("backend-test-windows", "scripts/ci_file_shard_pins_windows.txt"), + ): + steps = workflow["jobs"][name]["steps"] + names = [step.get("name", "") for step in steps] + prewarm = [i for i, n in enumerate(names) if n.startswith("Prewarm assertion rewrites")] + tests = [i for i, n in enumerate(names) if n.startswith("Run tests")] + assert len(prewarm) == 1 and len(tests) == 1, names + assert prewarm[0] + 1 == tests[0] + step, run_tests = steps[prewarm[0]], steps[tests[0]] + # The scope step writes reduced=false for the full suite and for nothing else. + assert step["if"] == "${{ steps.scope.outputs.reduced == 'false' }}" + # Same user, same working directory, same shard as the test step. + assert step.get("shell") == run_tests.get("shell") + assert "working-directory" not in step and "working-directory" not in run_tests + command = " ".join(step["run"].replace("\\\n", " ").split()) + assert command == ( + "python -m scripts.ci_rewrite_prewarm " + '--file-shards "$SHARD_COUNT" --file-shard ${{ matrix.group }} ' + f"--file-shard-pins {pins}" + ) + assert f"--file-shard-pins {pins}" in run_tests["run"] + # A cache-root override in either step would send the two to different trees. + for variable in ("XDG_CACHE_HOME", "PYTHONPYCACHEPREFIX", "PYTHONDONTWRITEBYTECODE"): + assert variable not in step.get("env", {}) + assert variable not in run_tests.get("env", {}) + assert step["continue-on-error"] is True + assert step["timeout-minutes"] <= 5 + # Bytecode is produced on the runner each run, never restored from a cache. + assert not [s for s in steps if "actions/cache" in s.get("uses", "")] + for name, job in workflow["jobs"].items(): + if name not in ("backend-test", "backend-test-windows"): + assert "ci_rewrite_prewarm" not in str(job)