Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions buildkite/pipeline_generator/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,12 @@ The generator relies on several environment variables, typically provided by Bui
* `VLLM_CI_KERNREC`: Set to "1" to load the kernel-launch recorder (`buildkite/ci_selector/recorders/kernrec`) into every CUDA process of every GPU step and upload its `.kernrec/**` output as job artifacts. Off by default; meant for nightly and post-merge builds that feed the test selector.
* `VLLM_CI_KERNREC_AMD`: Set to "1" to do the same on AMD steps with the ROCm backend. Independent of `VLLM_CI_KERNREC`: either backend can be recorded alone.
* `VLLM_CI_FNREC`: Set to "1" to load the Python function recorder (`buildkite/ci_selector/recorders/fnrec`) into every Python process of every step, writing to its own `.fnrec/<job-id>`. Off by default; independent of `VLLM_CI_KERNREC`, same audience.
* `VLLM_CI_INCLUDE_STEP_KEYS`: A non-empty JSON array of stable step keys to
add to the normal selection, including optional jobs and their dependencies.
For example, `["language-models-test-mteb"]`. Applies to this build only;
unknown or off-platform keys warn and discard the include request, keeping
the normal selection. Cannot be combined with
`VLLM_CI_ONLY_STEP_KEYS`. `NOAUTO=1` still defers all jobs.
* `VLLM_CI_ONLY_STEP_KEYS`: A non-empty JSON array of stable step keys. When
set, the generator emits those steps and their transitive dependencies,
ignoring normal source-file selection. Generated AMD mirror keys such as
Expand Down
18 changes: 11 additions & 7 deletions buildkite/pipeline_generator/buildkite_step.py
Original file line number Diff line number Diff line change
Expand Up @@ -721,7 +721,9 @@ def fnrec_collect_group(groups: "List[BuildkiteGroupStep]") -> "BuildkiteGroupSt

def _kernrec_backend(step: "BuildkiteCommandStep") -> str:
"""The backend the kernel recorder prelude exports on this rendered step."""
text = " ".join(list(step.commands or []) + [str(v) for v in (step.env or {}).values()])
text = " ".join(
list(step.commands or []) + [str(v) for v in (step.env or {}).values()]
)
return "rocm" if "KERNREC_BACKEND=rocm" in text else "cuda"


Expand Down Expand Up @@ -814,7 +816,11 @@ def _collect_step(
# since #685, and the step passed.
for backend in ("cuda", "rocm"):
env[f"KERNREC_EXPECTED_JOBS_{backend.upper()}"] = str(
sum(s.parallelism or 1 for s in armed if _kernrec_backend(s) == backend)
sum(
s.parallelism or 1
for s in armed
if _kernrec_backend(s) == backend
)
)
branch = os.getenv("VLLM_CI_BRANCH") or "main"
url = (
Expand Down Expand Up @@ -1111,11 +1117,7 @@ def _shard_at_generation(
if mode == "shadow":
runtime_shard.annotate(step_key, f"(shadow) {text}", "info")
return None
if (
result is None
or result["overBudget"]
or runtime_shard.setup_too_long(result)
):
if result is None or result["overBudget"] or runtime_shard.setup_too_long(result):
runtime_shard.annotate(step_key, text, "warning")
if result is None or result["shards"] < 2:
return None
Expand Down Expand Up @@ -1490,6 +1492,8 @@ def _step_should_run(step: Step, list_file_diff: List[str]) -> bool:
global_config = get_global_config()
if global_config["only_step_keys"] is not None:
return step.key in global_config["only_step_keys"]
if step.key in (global_config.get("include_step_keys") or ()):
return True
# The test selector's choice (selector_gate.py), prerequisites included.
# Unlike only_step_keys it removes nothing: the rest are deferred, so
# "Run more jobs" still offers them.
Expand Down
21 changes: 13 additions & 8 deletions buildkite/pipeline_generator/global_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@


ONLY_STEP_KEYS_ENV_VAR = "VLLM_CI_ONLY_STEP_KEYS"
INCLUDE_STEP_KEYS_ENV_VAR = "VLLM_CI_INCLUDE_STEP_KEYS"
PLATFORM_ENV_VAR = "VLLM_CI_PLATFORM"
STEP_KEY_PATTERN = re.compile(r"^[a-zA-Z0-9_-]+$")

Expand Down Expand Up @@ -37,6 +38,7 @@ class GlobalConfig(TypedDict):
merge_base_commit: Optional[str] = None
fail_fast: bool = False
only_step_keys: Optional[FrozenSet[str]] = None
include_step_keys: Optional[FrozenSet[str]] = None
platform: Optional[Platform] = None


Expand Down Expand Up @@ -96,7 +98,10 @@ def init_global_config(pipeline_config_path: str):
merge_base_commit=merge_base_commit,
list_file_diff=list_file_diff,
fail_fast=_should_fail_fast(pr_labels),
only_step_keys=_parse_only_step_keys(os.getenv(ONLY_STEP_KEYS_ENV_VAR)),
only_step_keys=_parse_step_keys(os.getenv(ONLY_STEP_KEYS_ENV_VAR)),
include_step_keys=_parse_step_keys(
os.getenv(INCLUDE_STEP_KEYS_ENV_VAR), INCLUDE_STEP_KEYS_ENV_VAR
),
platform=_parse_platform(
os.getenv(PLATFORM_ENV_VAR), os.getenv("TORCH_NIGHTLY") == "1"
),
Expand All @@ -116,24 +121,24 @@ def get_global_config():
return config


def _parse_only_step_keys(value: Optional[str]) -> Optional[FrozenSet[str]]:
def _parse_step_keys(
value: Optional[str], env_var: str = ONLY_STEP_KEYS_ENV_VAR
) -> Optional[FrozenSet[str]]:
if value is None:
return None
try:
parsed = json.loads(value)
except json.JSONDecodeError as error:
raise ValueError(
f"{ONLY_STEP_KEYS_ENV_VAR} must be a JSON array of step keys."
) from error
raise ValueError(f"{env_var} must be a JSON array of step keys.") from error
if not isinstance(parsed, list) or not parsed:
raise ValueError(f"{ONLY_STEP_KEYS_ENV_VAR} must be a non-empty JSON array.")
raise ValueError(f"{env_var} must be a non-empty JSON array.")
if any(
not isinstance(key, str) or not STEP_KEY_PATTERN.fullmatch(key)
for key in parsed
):
raise ValueError(f"{ONLY_STEP_KEYS_ENV_VAR} contains an invalid step key.")
raise ValueError(f"{env_var} contains an invalid step key.")
if len(set(parsed)) != len(parsed):
raise ValueError(f"{ONLY_STEP_KEYS_ENV_VAR} contains duplicate step keys.")
raise ValueError(f"{env_var} contains duplicate step keys.")
return frozenset(parsed)


Expand Down
65 changes: 49 additions & 16 deletions buildkite/pipeline_generator/pipeline_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,25 @@ def generate_images_only(self):

def generate(self):
global_config = get_global_config()
included = global_config.get("include_step_keys")
if included and global_config["only_step_keys"] is not None:
raise ValueError(
"VLLM_CI_INCLUDE_STEP_KEYS cannot be combined with VLLM_CI_ONLY_STEP_KEYS."
)

platform = global_config["platform"]
steps = []
for job_dir in global_config["job_dirs"]:
steps.extend(read_steps_from_job_dir(job_dir))
try:
_, included = select_steps_and_dependencies(steps, included, platform)
except ValueError as error:
_annotate_best_effort(
f"{error} Ignoring requested extra jobs; running normal CI.",
style="warning",
)
included = None
global_config["include_step_keys"] = included

# Skip if changes are doc-only (unless RUN_ALL is set)
if _skips_ci_as_docs_only(global_config):
Expand All @@ -128,7 +147,6 @@ def generate(self):
f.write("true")
return

platform = global_config["platform"]
if platform:
reason = (
"torch nightly validates CUDA only"
Expand All @@ -137,9 +155,6 @@ def generate(self):
)
_annotate_best_effort(f"Only {platform.value} jobs run: {reason}.")

steps = []
for job_dir in global_config["job_dirs"]:
steps.extend(read_steps_from_job_dir(job_dir))
all_steps = list(steps)
images_step = _images_step_key()
# With an images step, it uploads them; bootstrap only learns which,
Expand Down Expand Up @@ -186,6 +201,11 @@ def generate(self):
)
else:
record["scheduled_keys"] = record["legacy_keys"]
if included:
record["included_keys"] = sorted(included)
record["scheduled_keys"] = sorted(
set(record["scheduled_keys"]) | included
)
selector_gate.write_and_upload(
record, os.path.dirname(self.output_file_path)
)
Expand Down Expand Up @@ -266,11 +286,13 @@ def generate(self):

def _skips_ci_as_docs_only(global_config: dict) -> bool:
"""Whether the build runs nothing: a doc-only change, unless RUN_ALL or a
list of steps asks for them."""
list of steps asks for them (VLLM_CI_ONLY_STEP_KEYS, or the extra jobs of
`/ci run --include`)."""
return (
global_config["docs_only_disable"] == "0"
and not global_config["run_all"]
and global_config["only_step_keys"] is None
and not global_config.get("include_step_keys")
and is_docs_only_change(global_config["list_file_diff"])
)

Expand Down Expand Up @@ -490,17 +512,28 @@ def write_deferred_steps(

def _legacy_keys(steps: List[Step], diff: List[str]) -> List[str]:
"""The keys today's rules run, AMD mirrors included, decided the way
convert_group_step_to_buildkite_step decides them."""
keys = []
for step in steps:
if _step_should_run(step, diff):
keys.append(step.key)
amd = (step.mirror or {}).get("amd")
if amd and _amd_mirror_should_run(
_step_should_run(_get_amd_mirror_effective_step(step, amd), diff), diff
):
keys.append(f"amd-{step.key}")
return keys
convert_group_step_to_buildkite_step decides them.

Steps a `/ci run --include` request added are the requester's choice,
not today's rules: they stay out of this baseline, which the selector is
compared with, and are recorded as `included_keys` instead."""
global_config = get_global_config()
included = global_config.get("include_step_keys")
global_config["include_step_keys"] = None
try:
keys = []
for step in steps:
if _step_should_run(step, diff):
keys.append(step.key)
amd = (step.mirror or {}).get("amd")
if amd and _amd_mirror_should_run(
_step_should_run(_get_amd_mirror_effective_step(step, amd), diff),
diff,
):
keys.append(f"amd-{step.key}")
return keys
finally:
global_config["include_step_keys"] = included


# How selector_gate records the selector declining because it cannot narrow
Expand Down
4 changes: 4 additions & 0 deletions buildkite/pipeline_generator/selector_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,10 @@ def base_record(global_config: Dict[str, Any], gate_mode: str) -> Dict[str, Any]
"selector_keys": None,
"explain": None,
"legacy_keys": None,
# Steps added by `/ci run --include` (VLLM_CI_INCLUDE_STEP_KEYS) with
# their prerequisites: scheduled, but in neither the selector's keys
# nor today's rules.
"included_keys": None,
"scheduled_keys": None,
"records": None,
"seconds": None,
Expand Down
17 changes: 12 additions & 5 deletions buildkite/tests/pipeline_generator/test_global_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from buildkite.pipeline_generator.global_config import (
ONLY_STEP_KEYS_ENV_VAR,
_matches_run_all_patterns,
_parse_only_step_keys,
_parse_step_keys,
_parse_platform,
_validate_pipeline_config,
init_global_config,
Expand Down Expand Up @@ -91,8 +91,8 @@ def test_init_global_config_fork_branch(
# Should succeed


def test_parse_only_step_keys():
assert _parse_only_step_keys(
def test_parse_step_keys():
assert _parse_step_keys(
'["basic-models-test-other-cpu", "image-build-cpu"]'
) == frozenset(
{
Expand All @@ -111,9 +111,9 @@ def test_parse_only_step_keys():
('["same", "same"]', "contains duplicate step keys"),
],
)
def test_parse_only_step_keys_rejects_invalid_values(value, message):
def test_parse_step_keys_rejects_invalid_values(value, message):
with pytest.raises(ValueError, match=message):
_parse_only_step_keys(value)
_parse_step_keys(value)


@patch(
Expand Down Expand Up @@ -220,3 +220,10 @@ def test_amd_run_all_patterns_use_the_shared_excludes():
assert not match(["csrc/rocm/a.cu"], patterns, excludes)
assert not match(["vllm/a.py"], patterns, excludes)
assert not match(["csrc/a.cu"], None, excludes), "no patterns, no match"


def test_parse_step_keys_names_the_include_variable():
from buildkite.pipeline_generator.global_config import INCLUDE_STEP_KEYS_ENV_VAR

with pytest.raises(ValueError, match=INCLUDE_STEP_KEYS_ENV_VAR):
_parse_step_keys("[]", INCLUDE_STEP_KEYS_ENV_VAR)
95 changes: 95 additions & 0 deletions buildkite/tests/pipeline_generator/test_selector_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import subprocess

import pytest
import yaml

import selector_gate
from constants import Platform
Expand Down Expand Up @@ -437,3 +438,97 @@ def emission(**kw):
assert record["selector_changed_steps"] == ["nightly-changed"]
assert record["selector_keys"] == ["nightly-changed", "test-b"]
assert record["selector_optional_dropped"] == ["nightly-other"]


# `/ci run --include <key> ...` (VLLM_CI_INCLUDE_STEP_KEYS)


def _steps_with_optional():
return _steps() + [
Step(
label="Opt",
key="test-opt",
depends_on=["image-build"],
commands=["opt"],
source_file_dependencies=["vllm/"],
optional=True,
),
]


def _generate(config, monkeypatch, tmp_path, steps=_steps_with_optional):
monkeypatch.setattr(pg, "read_steps_from_job_dir", lambda _: steps())
monkeypatch.delenv("BUILDKITE_JOB_ID", raising=False)
monkeypatch.setenv("PATH", "") # no buildkite-agent: uploads are skipped
annotations = []
monkeypatch.setattr(
pg, "_annotate_best_effort", lambda msg, style=None: annotations.append(msg)
)
config["job_dirs"] = ["jobs"]
generator = object.__new__(pg.PipelineGenerator)
generator.output_file_path = str(tmp_path / "pipeline.yaml")
generator.generate()
return annotations


def _ready_keys(tmp_path):
"""Keys that run without anyone unblocking them."""
pipeline = yaml.safe_load(open(tmp_path / "pipeline.yaml"))
return {
step.get("key")
for group in pipeline["steps"]
for step in group.get("steps", [group])
if step.get("key")
}


def test_requested_jobs_run_beside_the_selectors(pr_config, monkeypatch, tmp_path):
"""An included optional step runs with the selector's choice, and is
recorded apart from today's rules, the baseline the selector is scored
against."""
monkeypatch.setattr(selector_gate, "run_ci_select", _emission(["test-b"]))
pr_config["include_step_keys"] = frozenset({"test-opt"})
_generate(pr_config, monkeypatch, tmp_path)

record = json.load(open(tmp_path / selector_gate.DECISION_FILE))
assert record["applied"] is True
assert record["legacy_keys"] == ["image-build", "test-a"]
assert record["included_keys"] == ["image-build", "test-opt"]
assert record["scheduled_keys"] == ["image-build", "test-b", "test-opt"]
assert {"test-b", "test-opt"} <= _ready_keys(tmp_path)
assert "test-a" not in _ready_keys(tmp_path)


def test_requested_jobs_stay_out_of_todays_rules(pr_config):
pr_config["include_step_keys"] = frozenset({"test-opt", "test-b"})
assert pg._legacy_keys(_steps_with_optional(), ["vllm/foo.py"]) == [
"image-build",
"test-a",
]
assert pr_config["include_step_keys"] == frozenset({"test-opt", "test-b"})


def test_an_unknown_requested_key_keeps_normal_ci(pr_config, monkeypatch, tmp_path):
monkeypatch.setattr(selector_gate, "run_ci_select", _emission(["test-b"]))
pr_config["include_step_keys"] = frozenset({"no-such-step"})
annotations = _generate(pr_config, monkeypatch, tmp_path)

assert any("Ignoring requested extra jobs" in a for a in annotations)
record = json.load(open(tmp_path / selector_gate.DECISION_FILE))
assert record["included_keys"] is None
assert record["scheduled_keys"] == ["image-build", "test-b"]
assert "test-opt" not in _ready_keys(tmp_path)


def test_requested_jobs_run_on_a_docs_only_change(pr_config, monkeypatch, tmp_path):
pr_config.update(docs_only_disable="0", list_file_diff=["docs/index.md"])
pr_config["include_step_keys"] = frozenset({"test-opt"})
_generate(pr_config, monkeypatch, tmp_path)
assert "test-opt" in _ready_keys(tmp_path)


def test_include_cannot_be_combined_with_only(pr_config, monkeypatch, tmp_path):
pr_config["include_step_keys"] = frozenset({"test-opt"})
pr_config["only_step_keys"] = frozenset({"test-a"})
with pytest.raises(ValueError, match="cannot be combined"):
_generate(pr_config, monkeypatch, tmp_path)
Loading