Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
52 commits
Select commit Hold shift + click to select a range
3a847a9
runtime-shard: check on every build that no test dropped out of its s…
Thangnguyenvn98 Oct 30, 2026
b4a5435
runtime-shard: list a test that two shards both ran, as a warning
Thangnguyenvn98 Nov 5, 2026
cc08303
runtime-shard: keep the coverage check whole when a shard dies or ret…
Thangnguyenvn98 Nov 5, 2026
bbe2e6f
runtime-shard: fail the coverage check when it can't prove every test…
Thangnguyenvn98 Nov 13, 2026
8c827e5
runtime-shard: collect the expected test list in a separate pytest
Thangnguyenvn98 Nov 13, 2026
53f9b5c
runtime-shard: an inventory subprocess that cannot start fails closed
Thangnguyenvn98 Nov 13, 2026
bb7eb38
runtime-shard: enforce the coverage check
Thangnguyenvn98 Nov 13, 2026
d27d788
runtime-shard: check the coverage of picked deferred steps
Thangnguyenvn98 Nov 13, 2026
5314a85
runtime-shard: let a coverage job pick its annotation context
Thangnguyenvn98 Nov 13, 2026
4bf52f3
runtime-shard: take the coverage annotation context from the step key
Thangnguyenvn98 Nov 13, 2026
f87e274
runtime-shard: clarify unreadable coverage records
Thangnguyenvn98 Nov 13, 2026
e7606fd
runtime-shard: soft-fail steps warn on gaps, only the build's own rec…
Thangnguyenvn98 Nov 13, 2026
3693b62
runtime-shard: bound the inventory and record its build
Thangnguyenvn98 Nov 13, 2026
aec2464
runtime-shard: send unplanned tests to the least-loaded owner
Thangnguyenvn98 Nov 13, 2026
8ecb7d7
runtime-shard: wait out the coverage checks without a deadline
Thangnguyenvn98 Nov 13, 2026
3a315bd
runtime-shard: keep deferred jobs out of shard coverage
Thangnguyenvn98 Nov 13, 2026
7b30c21
runtime-shard: fail closed on missing soft-fail evidence
Thangnguyenvn98 Nov 13, 2026
326047b
runtime-shard: handle coverage review edge cases
Thangnguyenvn98 Nov 13, 2026
efc042a
runtime-shard: test coverage diagnostic boundaries
Thangnguyenvn98 Nov 13, 2026
2a1b673
runtime-shard: pin inventory failure diagnostics
Thangnguyenvn98 Nov 13, 2026
b9761b0
runtime-shard: close dependency and attribution gaps
Thangnguyenvn98 Nov 13, 2026
8f9e1b0
runtime-shard: preserve unknown shard plans
Thangnguyenvn98 Nov 13, 2026
505509a
runtime-shard: test unowned-test attribution
Thangnguyenvn98 Nov 13, 2026
de6c0c0
runtime-shard: depend on shards for coverage
Thangnguyenvn98 Nov 13, 2026
e63de78
runtime-shard: preserve coverage after canceled shards
Thangnguyenvn98 Nov 14, 2026
aff4878
runtime-shard: keep inventory out of recorder data
Thangnguyenvn98 Nov 14, 2026
e47d2ef
runtime-shard: test empty soft-fail outcome
Thangnguyenvn98 Nov 14, 2026
18de06f
runtime-shard: close final coverage review gaps
Thangnguyenvn98 Nov 14, 2026
8132499
runtime-shard: bound coverage wait and preserve agent retries
Thangnguyenvn98 Nov 14, 2026
ac35d16
runtime-shard: test coverage and recorder wait behavior
Thangnguyenvn98 Nov 14, 2026
234f9e1
runtime-shard: reject incomplete coverage evidence
Thangnguyenvn98 Nov 14, 2026
7a857e1
runtime-shard: test malformed coverage record variants
Thangnguyenvn98 Nov 14, 2026
621f23b
runtime-shard: isolate malformed ran ID regression
Thangnguyenvn98 Nov 14, 2026
f571e74
runtime-shard: finish recorder and retry cleanup
Thangnguyenvn98 Nov 14, 2026
0f8d204
runtime-shard: preserve definite misses and retry guidance
Thangnguyenvn98 Nov 14, 2026
268e21d
runtime-shard: keep retry advice for retryable gaps
Thangnguyenvn98 Nov 14, 2026
7071517
runtime-shard: test retry advice for each missing-evidence kind
Thangnguyenvn98 Nov 14, 2026
dfe9085
runtime-shard: isolate analytics and structure retry advice
Thangnguyenvn98 Nov 14, 2026
5e1027a
runtime-shard: test retry advice across mixed outcomes
Thangnguyenvn98 Nov 14, 2026
ad26795
runtime-shard: report misses by command
Thangnguyenvn98 Nov 14, 2026
e7a20cc
runtime-shard: handle collection and artifact failures
Thangnguyenvn98 Nov 14, 2026
932af1d
runtime-shard: isolate inventory coverage data
Thangnguyenvn98 Nov 14, 2026
560466b
runtime-shard: use valid inventory across owner attempts
Thangnguyenvn98 Nov 14, 2026
fadcc90
runtime-shard: test inventory boundaries and ordering
Thangnguyenvn98 Nov 14, 2026
c54f99e
runtime-shard: retry artifact download failures
Thangnguyenvn98 Nov 14, 2026
e7b1aed
runtime-shard: assert download failure reporting
Thangnguyenvn98 Nov 14, 2026
024617e
runtime-shard: treat empty artifact downloads as missing evidence
Thangnguyenvn98 Nov 14, 2026
bf6facc
runtime-shard: install the coverage checker
Thangnguyenvn98 Nov 14, 2026
557f62b
runtime-shard: handle inventory and wait timeouts safely
Thangnguyenvn98 Nov 14, 2026
ffbb236
runtime-shard: retry downloads and verify terminal outcomes
Thangnguyenvn98 Nov 14, 2026
f98604d
runtime-shard: close final coverage and cleanup gaps
Thangnguyenvn98 Nov 14, 2026
c22b299
runtime-shard: unify inventory cleanup and miss attribution
Thangnguyenvn98 Nov 14, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/pipeline-generator-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,5 +40,8 @@ jobs:
python -m pip install -r requirements.txt
python -m pip install pytest ./buildkite/pipeline_generator

- name: Check installed generator
run: python -I -c 'import runtime_shard_coverage'

- name: Run tests
run: python -m pytest buildkite/tests
163 changes: 141 additions & 22 deletions buildkite/pipeline_generator/buildkite_step.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from pydantic import BaseModel, Field
from typing import Dict, List, Optional, Any, Literal, Tuple
from copy import deepcopy
import json
import math
import os
import time
Expand Down Expand Up @@ -794,25 +795,58 @@ def _carries(step: "BuildkiteCommandStep", recorder: str) -> bool:
# tails are blocked or hung jobs.
COLLECT_AFTER_MINUTES = 180

# Polls each recording step's state through the agent until all have finished
# or the deadline passes. `$$`: Buildkite would interpolate a bare `$` at
# upload time, in the bootstrap's environment. An unreadable state (an older
# agent, a step that is not there) reads as unfinished, so the step waits to
# the deadline rather than collecting early.
COLLECT_WAIT = (
'deadline="$$CI_SELECTOR_COLLECT_DEADLINE"; '
'while [ "$$(date +%s)" -lt "$$deadline" ]; do '
'pending=""; '
"for k in $$CI_SELECTOR_COLLECT_WAIT_FOR; do "
's="$$(buildkite-agent step get state --step "$$k" 2>/dev/null)"; '
'case "$$s" in finished|ignored) ;; *) pending="$$k"; break;; esac; '
"done; "
'[ -z "$$pending" ] && { echo "every recording step has finished"; break; }; '
'echo "waiting on $$pending ($${s:-state unknown}); collecting at the latest at '
'$$(date -u -d "@$$deadline" +%H:%MZ)"; '
"sleep 300; "
"done"
def _wait_for_steps(
terminal_states: str,
sleep_seconds: int,
deadline_env: str,
finished_message: str,
mark_deadline: bool = False,
) -> str:
"""Poll step states, then let the caller handle anything still unfinished."""
marker = (
'marker="$${TMPDIR:-/tmp}/runtime-shard-coverage-'
'$$BUILDKITE_JOB_ID-wait-timed-out"; rm -f "$$marker"; '
if mark_deadline
else ""
)
touch = 'touch "$$marker"; ' if mark_deadline else ""
return (
f'{marker}deadline="$${deadline_env}"; '
"while :; do "
'pending=""; '
"for k in $$CI_SELECTOR_COLLECT_WAIT_FOR; do "
's="$$(buildkite-agent step get state --step "$$k" 2>/dev/null)"; '
f'case "$$s" in {terminal_states}) ;; '
'*) pending="$$k"; break;; esac; '
"done; "
f'[ -z "$$pending" ] && {{ echo "{finished_message}"; break; }}; '
'[ "$$(date +%s)" -ge "$$deadline" ] && '
f'{{ {touch}echo "wait deadline reached with $$pending '
'($${s:-state unknown}); continuing"; break; }; '
'echo "waiting on $$pending ($${s:-state unknown}); deadline '
'$$(date -u -d "@$$deadline" +%H:%MZ)"; '
f"sleep {sleep_seconds}; "
"done"
)


# At the deadline, collection folds the available recordings and coverage
# checks the available evidence. Coverage still fails anything left unproven.
COLLECT_WAIT = _wait_for_steps(
"finished|ignored",
300,
"CI_SELECTOR_COLLECT_DEADLINE",
"every recording step has finished",
)
COVERAGE_WAIT = _wait_for_steps(
"finished|ignored|canceled|waiting_failed",
30,
"RUNTIME_SHARD_COVERAGE_DEADLINE",
"every sharded step has stopped",
mark_deadline=True,
)
COVERAGE_TIMEOUT_MINUTES = 360
COVERAGE_WAIT_MINUTES = COVERAGE_TIMEOUT_MINUTES - 15


def _collect_step(
Expand Down Expand Up @@ -900,6 +934,81 @@ def _collect_step(
)


def shard_coverage_step(
groups: "List[BuildkiteGroupStep]",
) -> "Optional[BuildkiteCommandStep]":
"""The job that checks no test dropped out of a build's shards, or None
when nothing in this build is sharded.

Fails when a test is missed or coverage cannot be proven, so a successful
shard cannot hide a missing record. The comparison source is packed at
generation, so the job runs the same revision as the plugin without a
runtime download. Polling rather than depending on the shards means a
canceled soft-fail shard cannot suppress the whole check. At the wait
deadline, the checker runs and fails any coverage still unproven, including
a soft-fail step whose final outcome is not yet known.
"""
steps = {}
sharded = []
for group in groups:
for step in group.steps:
packed = (step.env or {}).get(runtime_shard.PLAN_ENV)
if not packed:
continue
plan = runtime_shard.unpack(packed)
sharded.append(step)
steps[step.key] = [
{
"position": planned["position"],
"label": planned["label"],
"fallback": planned["fallback"],
"owners": runtime_shard.owners(planned),
# Only a terminal failure on a soft-fail step relaxes gaps;
# a passed step still has to prove coverage.
"soft_fail": bool(step.soft_fail),
}
for planned in plan["commands"]
]
if not steps:
return None
source_path = os.path.join(
os.path.dirname(__file__), "runtime_shard_coverage.py"
)
with open(source_path) as f:
coverage_source = f.read()
on_main = get_global_config()["branch"] == "main"
queue = AgentQueue.SMALL_CPU_POSTMERGE if on_main else AgentQueue.SMALL_CPU_PREMERGE
retry = ensure_infra_failure_retry(None)
depends_on = []
for dep in (d for s in sharded for d in (s.depends_on or [])):
if dep not in depends_on:
depends_on.append(dep)
return BuildkiteCommandStep(
label=":mag: Check every test ran",
key="runtime-shard-coverage",
env={
"RUNTIME_SHARD_COVERAGE_STEPS": json.dumps(steps),
"CI_SELECTOR_COLLECT_WAIT_FOR": " ".join(steps),
"RUNTIME_SHARD_COVERAGE_DEADLINE": str(
int(time.time()) + COVERAGE_WAIT_MINUTES * 60
),
"RUNTIME_SHARD_COVERAGE_SCRIPT": runtime_shard.pack(coverage_source),
},
agents={"queue": queue.value},
commands=[
COVERAGE_WAIT,
'python3 -I -c \'import base64,json,os,zlib;'
'exec(json.loads(zlib.decompress(base64.b64decode('
'os.environ["RUNTIME_SHARD_COVERAGE_SCRIPT"]))))\'',
],
depends_on=depends_on,
allow_dependency_failure=True,
soft_fail=False,
timeout_in_minutes=COVERAGE_TIMEOUT_MINUTES,
retry=retry,
)


def _kernrec_finish_command() -> str:
"""Record the step's exit status in the recorder's sidecar, explicitly.

Expand Down Expand Up @@ -1235,6 +1344,7 @@ def _shard_at_generation(
commands: List[str],
env: Optional[Dict[str, str]],
mode: str,
setup_profile: SetupProfile = "nvidia",
) -> Optional[Tuple[int, List[str], Dict[str, str]]]:
"""Plan an enrolled step's shards from the checkout (runtime_shard.py)
and, in "on" mode, turn its job's prepared commands and env into one
Expand Down Expand Up @@ -1270,7 +1380,7 @@ def _shard_at_generation(
f"{env.get('PYTEST_ADDOPTS', '')} -p no:pytest-shard"
f" -p {runtime_shard.PLUGIN_MODULE}"
).strip()
plan = {"shards": result["shards"], "commands": result["commands"]}
plan = {"shards": result["shards"], "commands": result["commands"], "key": step_key}
env[runtime_shard.PLAN_ENV] = runtime_shard.pack(plan)
env[runtime_shard.PLUGIN_ENV] = runtime_shard.plugin_source()
commands = list(commands)
Expand All @@ -1290,7 +1400,8 @@ def _shard_at_generation(
commands[at + 1] = runtime_shard.guard(
commands[at + 1], planned, result["shards"]
)
return result["shards"], runtime_shard.shard_job_setup() + commands, env
checkout = _fnrec_checkout_path(step, setup_profile)
return result["shards"], runtime_shard.shard_job_setup(checkout) + commands, env


def convert_group_step_to_buildkite_step(
Expand Down Expand Up @@ -1345,6 +1456,7 @@ def convert_group_step_to_buildkite_step(
amd_commands,
amd_env,
amd_shard_mode,
"amd",
)
if amd_shard_mode
else None
Expand Down Expand Up @@ -1373,7 +1485,8 @@ def convert_group_step_to_buildkite_step(
extra_artifact_paths=_fnrec_artifact_paths(step, "amd")
+ (
[KERNREC_ARTIFACT_PATH] if _kernrec_applies(step, "amd") else []
),
)
+ ([runtime_shard.COVERAGE_ARTIFACT_PATH] if amd_sharded else []),
recorded_step=step,
)
amd_step.deferred = not _step_should_run(step, list_file_diff)
Expand Down Expand Up @@ -1493,6 +1606,10 @@ def convert_group_step_to_buildkite_step(
) = sharded
# Buildkite fills in %N (from 1) and %t (the shard count): "... Shard 2/4".
buildkite_step.label = f"{buildkite_step.label} Shard %N/%t"
buildkite_step.artifact_paths = _merge_artifact_paths(
buildkite_step.artifact_paths,
[runtime_shard.COVERAGE_ARTIFACT_PATH],
)
if include_step:
group_steps_list.append(buildkite_step)

Expand Down Expand Up @@ -1566,6 +1683,7 @@ def convert_group_step_to_buildkite_step(
amd_commands,
extra_env,
amd_shard_mode,
"amd",
)
if amd_shard_mode
else None
Expand Down Expand Up @@ -1603,7 +1721,8 @@ def convert_group_step_to_buildkite_step(
[KERNREC_ARTIFACT_PATH]
if _kernrec_applies(amd_command_step, "amd")
else []
),
)
+ ([runtime_shard.COVERAGE_ARTIFACT_PATH] if amd_sharded else []),
recorded_step=amd_command_step,
)
amd_step.deferred = not _amd_mirror_should_run(
Expand Down
4 changes: 4 additions & 0 deletions buildkite/pipeline_generator/pipeline_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
convert_group_step_to_buildkite_step,
fnrec_collect_group,
one_job_annotation,
shard_coverage_step,
kernrec_collect_group,
kernrec_amd_enabled,
kernrec_enabled,
Expand Down Expand Up @@ -216,6 +217,9 @@ def generate(self):
one_job = one_job_annotation(buildkite_group_steps)
if one_job:
runtime_shard.annotate("one-job", one_job, "warning")
coverage = shard_coverage_step(buildkite_group_steps)
if coverage:
collect.append(coverage)
if collect:
buildkite_group_steps.append(
BuildkiteGroupStep(group=COLLECT_GROUP, steps=collect)
Expand Down
1 change: 1 addition & 0 deletions buildkite/pipeline_generator/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ py-modules = [
"amd",
"runtime_shard",
"runtime_shard_plugin",
"runtime_shard_coverage",
"unblock_jobs",
"selector_gate",
]
Expand Down
25 changes: 20 additions & 5 deletions buildkite/pipeline_generator/runtime_shard.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,10 @@
PLUGIN_ENV = "RUNTIME_SHARD_PLUGIN"
PLUGIN_DIR = "/tmp/runtime-shard"
PLUGIN_MODULE = "runtime_shard_plugin"
COVERAGE_DIR_ENV = "RUNTIME_SHARD_COVERAGE_DIR"
# Where a shard's coverage records go, relative to the checkout.
COVERAGE_DIR = ".runtime-shard-coverage"
COVERAGE_ARTIFACT_PATH = f"{COVERAGE_DIR}/*/*.json"
# pytest-shard would select a subset a second time on top of the plan.
# -n: pytest-xdist workers don't get the plan, so each shard would run it all.
_UNSHARDABLE_ARGS = ("--num-shards", "--shard-id", "-n", "--numprocesses")
Expand Down Expand Up @@ -780,9 +784,11 @@ def plan_step(
planned["prefix"] = prefix
planned["label"] = f"Command ({position + 1}/{len(commands)})"
planned["position"] = position
# A test the plan doesn't name runs in the first shard that
# runs the command at all.
planned["fallback"] = owners(planned)[0]
# A test the plan doesn't name runs in the least-loaded shard
# that runs the command at all; that shard also collects the
# command's full inventory, a cost shard 0 shouldn't always pay.
runs = owners(planned)
planned["fallback"] = min(runs, key=result["estimates"].__getitem__)
except Exception as error: # never block the step on its sharding
result, reason = None, str(error)
return result, summary(step_key, result, reason)
Expand Down Expand Up @@ -839,9 +845,17 @@ def plugin_source() -> str:
return base64.b64encode(zlib.compress(f.read().encode())).decode()


def shard_job_setup() -> List[str]:
def shard_job_setup(checkout: str) -> List[str]:
"""Commands that install the plugin in a shard job, before its own. $$ is
a literal $ in the uploaded pipeline."""
a literal $ in the uploaded pipeline.

Args:
checkout: Where this job can see the Buildkite checkout, the only
place artifact_paths can upload the coverage records from.

Returns:
The commands.
"""
unpack_plugin = (
"import base64, os, sys, zlib; sys.stdout.write(zlib.decompress("
f"base64.b64decode(os.environ['{PLUGIN_ENV}'])).decode())"
Expand All @@ -855,6 +869,7 @@ def shard_job_setup() -> List[str]:
# command that sets its own PYTHONPATH (PYTHONPATH=/vllm-workspace
# pytest ...) doesn't replace. It exits 0: a failed copy only warns.
f"python3 {PLUGIN_DIR}/{PLUGIN_MODULE}.py",
f'export {COVERAGE_DIR_ENV}="{checkout}/{COVERAGE_DIR}"',
]


Expand Down
Loading