From 3c302f518cb42c5a3e2ad4f54a8ba7d8d282d476 Mon Sep 17 00:00:00 2001 From: Taneem Ibrahim Date: Thu, 12 Nov 2026 01:45:01 -0500 Subject: [PATCH 1/3] Add opt-in CI jobs without replacing normal selection Signed-off-by: Taneem Ibrahim --- buildkite/pipeline_generator/README.md | 5 +++++ .../pipeline_generator/buildkite_step.py | 18 +++++++++++------- buildkite/pipeline_generator/global_config.py | 19 ++++++++++++------- .../pipeline_generator/pipeline_generator.py | 16 ++++++++++++++++ 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/buildkite/pipeline_generator/README.md b/buildkite/pipeline_generator/README.md index 05cf5940d..2594d074c 100644 --- a/buildkite/pipeline_generator/README.md +++ b/buildkite/pipeline_generator/README.md @@ -106,6 +106,11 @@ The generator relies on several environment variables, typically provided by Bui * `VLLM_CI_KERNREC`: Set to "1" to load the kernel-launch recorder (`buildkite/ci_selector/recorders/kernrec`) into every CUDA process of every GPU step and upload its `.kernrec/**` output as job artifacts. Off by default; meant for nightly and post-merge builds that feed the test selector. * `VLLM_CI_KERNREC_AMD`: Set to "1" to do the same on AMD steps with the ROCm backend. Independent of `VLLM_CI_KERNREC`: either backend can be recorded alone. * `VLLM_CI_FNREC`: Set to "1" to load the Python function recorder (`buildkite/ci_selector/recorders/fnrec`) into every Python process of every step, writing to its own `.fnrec/`. Off by default; independent of `VLLM_CI_KERNREC`, same audience. +* `VLLM_CI_INCLUDE_STEP_KEYS`: A non-empty JSON array of stable step keys to + add to the normal selection, including optional jobs and their dependencies. + For example, `["language-models-test-mteb"]`. Applies to this build only; + unknown or off-platform keys fail generation. Cannot be combined with + `VLLM_CI_ONLY_STEP_KEYS`. `NOAUTO=1` still defers all jobs. * `VLLM_CI_ONLY_STEP_KEYS`: A non-empty JSON array of stable step keys. When set, the generator emits those steps and their transitive dependencies, ignoring normal source-file selection. Generated AMD mirror keys such as diff --git a/buildkite/pipeline_generator/buildkite_step.py b/buildkite/pipeline_generator/buildkite_step.py index 0f901d073..49c014b3a 100644 --- a/buildkite/pipeline_generator/buildkite_step.py +++ b/buildkite/pipeline_generator/buildkite_step.py @@ -721,7 +721,9 @@ def fnrec_collect_group(groups: "List[BuildkiteGroupStep]") -> "BuildkiteGroupSt def _kernrec_backend(step: "BuildkiteCommandStep") -> str: """The backend the kernel recorder prelude exports on this rendered step.""" - text = " ".join(list(step.commands or []) + [str(v) for v in (step.env or {}).values()]) + text = " ".join( + list(step.commands or []) + [str(v) for v in (step.env or {}).values()] + ) return "rocm" if "KERNREC_BACKEND=rocm" in text else "cuda" @@ -814,7 +816,11 @@ def _collect_step( # since #685, and the step passed. for backend in ("cuda", "rocm"): env[f"KERNREC_EXPECTED_JOBS_{backend.upper()}"] = str( - sum(s.parallelism or 1 for s in armed if _kernrec_backend(s) == backend) + sum( + s.parallelism or 1 + for s in armed + if _kernrec_backend(s) == backend + ) ) branch = os.getenv("VLLM_CI_BRANCH") or "main" url = ( @@ -1111,11 +1117,7 @@ def _shard_at_generation( if mode == "shadow": runtime_shard.annotate(step_key, f"(shadow) {text}", "info") return None - if ( - result is None - or result["overBudget"] - or runtime_shard.setup_too_long(result) - ): + if result is None or result["overBudget"] or runtime_shard.setup_too_long(result): runtime_shard.annotate(step_key, text, "warning") if result is None or result["shards"] < 2: return None @@ -1490,6 +1492,8 @@ def _step_should_run(step: Step, list_file_diff: List[str]) -> bool: global_config = get_global_config() if global_config["only_step_keys"] is not None: return step.key in global_config["only_step_keys"] + if step.key in (global_config.get("include_step_keys") or ()): + return True # The test selector's choice (selector_gate.py), prerequisites included. # Unlike only_step_keys it removes nothing: the rest are deferred, so # "Run more jobs" still offers them. diff --git a/buildkite/pipeline_generator/global_config.py b/buildkite/pipeline_generator/global_config.py index af515abae..d4aac5592 100644 --- a/buildkite/pipeline_generator/global_config.py +++ b/buildkite/pipeline_generator/global_config.py @@ -10,6 +10,7 @@ ONLY_STEP_KEYS_ENV_VAR = "VLLM_CI_ONLY_STEP_KEYS" +INCLUDE_STEP_KEYS_ENV_VAR = "VLLM_CI_INCLUDE_STEP_KEYS" PLATFORM_ENV_VAR = "VLLM_CI_PLATFORM" STEP_KEY_PATTERN = re.compile(r"^[a-zA-Z0-9_-]+$") @@ -37,6 +38,7 @@ class GlobalConfig(TypedDict): merge_base_commit: Optional[str] = None fail_fast: bool = False only_step_keys: Optional[FrozenSet[str]] = None + include_step_keys: Optional[FrozenSet[str]] = None platform: Optional[Platform] = None @@ -97,6 +99,9 @@ def init_global_config(pipeline_config_path: str): list_file_diff=list_file_diff, fail_fast=_should_fail_fast(pr_labels), only_step_keys=_parse_only_step_keys(os.getenv(ONLY_STEP_KEYS_ENV_VAR)), + include_step_keys=_parse_only_step_keys( + os.getenv(INCLUDE_STEP_KEYS_ENV_VAR), INCLUDE_STEP_KEYS_ENV_VAR + ), platform=_parse_platform( os.getenv(PLATFORM_ENV_VAR), os.getenv("TORCH_NIGHTLY") == "1" ), @@ -116,24 +121,24 @@ def get_global_config(): return config -def _parse_only_step_keys(value: Optional[str]) -> Optional[FrozenSet[str]]: +def _parse_only_step_keys( + value: Optional[str], env_var: str = ONLY_STEP_KEYS_ENV_VAR +) -> Optional[FrozenSet[str]]: if value is None: return None try: parsed = json.loads(value) except json.JSONDecodeError as error: - raise ValueError( - f"{ONLY_STEP_KEYS_ENV_VAR} must be a JSON array of step keys." - ) from error + raise ValueError(f"{env_var} must be a JSON array of step keys.") from error if not isinstance(parsed, list) or not parsed: - raise ValueError(f"{ONLY_STEP_KEYS_ENV_VAR} must be a non-empty JSON array.") + raise ValueError(f"{env_var} must be a non-empty JSON array.") if any( not isinstance(key, str) or not STEP_KEY_PATTERN.fullmatch(key) for key in parsed ): - raise ValueError(f"{ONLY_STEP_KEYS_ENV_VAR} contains an invalid step key.") + raise ValueError(f"{env_var} contains an invalid step key.") if len(set(parsed)) != len(parsed): - raise ValueError(f"{ONLY_STEP_KEYS_ENV_VAR} contains duplicate step keys.") + raise ValueError(f"{env_var} contains duplicate step keys.") return frozenset(parsed) diff --git a/buildkite/pipeline_generator/pipeline_generator.py b/buildkite/pipeline_generator/pipeline_generator.py index 9996f2bba..477226915 100644 --- a/buildkite/pipeline_generator/pipeline_generator.py +++ b/buildkite/pipeline_generator/pipeline_generator.py @@ -65,12 +65,18 @@ def __init__( def generate(self): global_config = get_global_config() + included = global_config.get("include_step_keys") + if included and global_config["only_step_keys"] is not None: + raise ValueError( + "VLLM_CI_INCLUDE_STEP_KEYS cannot be combined with VLLM_CI_ONLY_STEP_KEYS." + ) # Skip if changes are doc-only (unless RUN_ALL is set) if ( global_config["docs_only_disable"] == "0" and not global_config["run_all"] and global_config["only_step_keys"] is None + and not included ): if is_docs_only_change(global_config["list_file_diff"]): print("List file diff: ", global_config["list_file_diff"]) @@ -100,6 +106,12 @@ def generate(self): steps = [] for job_dir in global_config["job_dirs"]: steps.extend(read_steps_from_job_dir(job_dir)) + try: + _, included = select_steps_and_dependencies(steps, included, platform) + except ValueError as error: + _annotate_best_effort(str(error), style="error") + raise + global_config["include_step_keys"] = included early_keys = upload_image_steps_early( steps, os.path.dirname(self.output_file_path) ) @@ -140,6 +152,10 @@ def generate(self): ) else: record["scheduled_keys"] = record["legacy_keys"] + if included: + record["scheduled_keys"] = sorted( + set(record["scheduled_keys"]) | included + ) selector_gate.write_and_upload( record, os.path.dirname(self.output_file_path) ) From 81581831820241191277367ff9179c3f46e6c918 Mon Sep 17 00:00:00 2001 From: Taneem Ibrahim Date: Thu, 12 Nov 2026 12:01:20 -0500 Subject: [PATCH 2/3] Keep normal CI selection when optional job keys are invalid Signed-off-by: Taneem Ibrahim --- buildkite/pipeline_generator/README.md | 3 ++- buildkite/pipeline_generator/global_config.py | 6 ++--- .../pipeline_generator/pipeline_generator.py | 24 +++++++++++-------- .../pipeline_generator/test_global_config.py | 10 ++++---- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/buildkite/pipeline_generator/README.md b/buildkite/pipeline_generator/README.md index 2594d074c..7c9c40701 100644 --- a/buildkite/pipeline_generator/README.md +++ b/buildkite/pipeline_generator/README.md @@ -109,7 +109,8 @@ The generator relies on several environment variables, typically provided by Bui * `VLLM_CI_INCLUDE_STEP_KEYS`: A non-empty JSON array of stable step keys to add to the normal selection, including optional jobs and their dependencies. For example, `["language-models-test-mteb"]`. Applies to this build only; - unknown or off-platform keys fail generation. Cannot be combined with + unknown or off-platform keys warn and discard the include request, keeping + the normal selection. Cannot be combined with `VLLM_CI_ONLY_STEP_KEYS`. `NOAUTO=1` still defers all jobs. * `VLLM_CI_ONLY_STEP_KEYS`: A non-empty JSON array of stable step keys. When set, the generator emits those steps and their transitive dependencies, diff --git a/buildkite/pipeline_generator/global_config.py b/buildkite/pipeline_generator/global_config.py index d4aac5592..9ae5f9c80 100644 --- a/buildkite/pipeline_generator/global_config.py +++ b/buildkite/pipeline_generator/global_config.py @@ -98,8 +98,8 @@ def init_global_config(pipeline_config_path: str): merge_base_commit=merge_base_commit, list_file_diff=list_file_diff, fail_fast=_should_fail_fast(pr_labels), - only_step_keys=_parse_only_step_keys(os.getenv(ONLY_STEP_KEYS_ENV_VAR)), - include_step_keys=_parse_only_step_keys( + only_step_keys=_parse_step_keys(os.getenv(ONLY_STEP_KEYS_ENV_VAR)), + include_step_keys=_parse_step_keys( os.getenv(INCLUDE_STEP_KEYS_ENV_VAR), INCLUDE_STEP_KEYS_ENV_VAR ), platform=_parse_platform( @@ -121,7 +121,7 @@ def get_global_config(): return config -def _parse_only_step_keys( +def _parse_step_keys( value: Optional[str], env_var: str = ONLY_STEP_KEYS_ENV_VAR ) -> Optional[FrozenSet[str]]: if value is None: diff --git a/buildkite/pipeline_generator/pipeline_generator.py b/buildkite/pipeline_generator/pipeline_generator.py index 477226915..f7f1c6ec7 100644 --- a/buildkite/pipeline_generator/pipeline_generator.py +++ b/buildkite/pipeline_generator/pipeline_generator.py @@ -71,6 +71,20 @@ def generate(self): "VLLM_CI_INCLUDE_STEP_KEYS cannot be combined with VLLM_CI_ONLY_STEP_KEYS." ) + platform = global_config["platform"] + steps = [] + for job_dir in global_config["job_dirs"]: + steps.extend(read_steps_from_job_dir(job_dir)) + try: + _, included = select_steps_and_dependencies(steps, included, platform) + except ValueError as error: + _annotate_best_effort( + f"{error} Ignoring requested extra jobs; running normal CI.", + style="warning", + ) + included = None + global_config["include_step_keys"] = included + # Skip if changes are doc-only (unless RUN_ALL is set) if ( global_config["docs_only_disable"] == "0" @@ -94,7 +108,6 @@ def generate(self): f.write("true") return - platform = global_config["platform"] if platform: reason = ( "torch nightly validates CUDA only" @@ -103,15 +116,6 @@ def generate(self): ) _annotate_best_effort(f"Only {platform.value} jobs run: {reason}.") - steps = [] - for job_dir in global_config["job_dirs"]: - steps.extend(read_steps_from_job_dir(job_dir)) - try: - _, included = select_steps_and_dependencies(steps, included, platform) - except ValueError as error: - _annotate_best_effort(str(error), style="error") - raise - global_config["include_step_keys"] = included early_keys = upload_image_steps_early( steps, os.path.dirname(self.output_file_path) ) diff --git a/buildkite/tests/pipeline_generator/test_global_config.py b/buildkite/tests/pipeline_generator/test_global_config.py index b129b3bbd..84398d863 100644 --- a/buildkite/tests/pipeline_generator/test_global_config.py +++ b/buildkite/tests/pipeline_generator/test_global_config.py @@ -6,7 +6,7 @@ from buildkite.pipeline_generator.global_config import ( ONLY_STEP_KEYS_ENV_VAR, _matches_run_all_patterns, - _parse_only_step_keys, + _parse_step_keys, _parse_platform, _validate_pipeline_config, init_global_config, @@ -91,8 +91,8 @@ def test_init_global_config_fork_branch( # Should succeed -def test_parse_only_step_keys(): - assert _parse_only_step_keys( +def test_parse_step_keys(): + assert _parse_step_keys( '["basic-models-test-other-cpu", "image-build-cpu"]' ) == frozenset( { @@ -111,9 +111,9 @@ def test_parse_only_step_keys(): ('["same", "same"]', "contains duplicate step keys"), ], ) -def test_parse_only_step_keys_rejects_invalid_values(value, message): +def test_parse_step_keys_rejects_invalid_values(value, message): with pytest.raises(ValueError, match=message): - _parse_only_step_keys(value) + _parse_step_keys(value) @patch( From ed6f1f68ab80333a9243e4e0e69abd0447e59a91 Mon Sep 17 00:00:00 2001 From: khluu Date: Fri, 13 Nov 2026 09:21:59 +0000 Subject: [PATCH 3/3] Keep included jobs out of the selector's baseline; add include tests legacy_keys is what today's rules run, the baseline the selector is scored against. _step_should_run now returns True for included keys, so every /ci run --include build counted the requested jobs as today's rules. Compute the baseline without them and record them separately as included_keys. Tests cover: included jobs run beside the selector's choice, stay out of legacy_keys, an unknown key keeps normal CI, the docs-only skip is bypassed, include + only is rejected, NOAUTO still defers, and the parse error names the include variable. Co-Authored-By: Claude Opus 5.5 Signed-off-by: khluu --- .../pipeline_generator/pipeline_generator.py | 34 ++++--- buildkite/pipeline_generator/selector_gate.py | 4 + .../pipeline_generator/test_global_config.py | 7 ++ .../pipeline_generator/test_selector_gate.py | 95 +++++++++++++++++++ .../tests/pipeline_generator/test_step.py | 20 ++++ 5 files changed, 149 insertions(+), 11 deletions(-) diff --git a/buildkite/pipeline_generator/pipeline_generator.py b/buildkite/pipeline_generator/pipeline_generator.py index f7f1c6ec7..3e690e2d6 100644 --- a/buildkite/pipeline_generator/pipeline_generator.py +++ b/buildkite/pipeline_generator/pipeline_generator.py @@ -157,6 +157,7 @@ def generate(self): else: record["scheduled_keys"] = record["legacy_keys"] if included: + record["included_keys"] = sorted(included) record["scheduled_keys"] = sorted( set(record["scheduled_keys"]) | included ) @@ -359,17 +360,28 @@ def write_deferred_steps( def _legacy_keys(steps: List[Step], diff: List[str]) -> List[str]: """The keys today's rules run, AMD mirrors included, decided the way - convert_group_step_to_buildkite_step decides them.""" - keys = [] - for step in steps: - if _step_should_run(step, diff): - keys.append(step.key) - amd = (step.mirror or {}).get("amd") - if amd and _amd_mirror_should_run( - _step_should_run(_get_amd_mirror_effective_step(step, amd), diff), diff - ): - keys.append(f"amd-{step.key}") - return keys + convert_group_step_to_buildkite_step decides them. + + Steps a `/ci run --include` request added are the requester's choice, + not today's rules: they stay out of this baseline, which the selector is + compared with, and are recorded as `included_keys` instead.""" + global_config = get_global_config() + included = global_config.get("include_step_keys") + global_config["include_step_keys"] = None + try: + keys = [] + for step in steps: + if _step_should_run(step, diff): + keys.append(step.key) + amd = (step.mirror or {}).get("amd") + if amd and _amd_mirror_should_run( + _step_should_run(_get_amd_mirror_effective_step(step, amd), diff), + diff, + ): + keys.append(f"amd-{step.key}") + return keys + finally: + global_config["include_step_keys"] = included # How selector_gate records the selector declining because it cannot narrow diff --git a/buildkite/pipeline_generator/selector_gate.py b/buildkite/pipeline_generator/selector_gate.py index c31d90efa..fab47336f 100644 --- a/buildkite/pipeline_generator/selector_gate.py +++ b/buildkite/pipeline_generator/selector_gate.py @@ -127,6 +127,10 @@ def base_record(global_config: Dict[str, Any], gate_mode: str) -> Dict[str, Any] "selector_keys": None, "explain": None, "legacy_keys": None, + # Steps added by `/ci run --include` (VLLM_CI_INCLUDE_STEP_KEYS) with + # their prerequisites: scheduled, but in neither the selector's keys + # nor today's rules. + "included_keys": None, "scheduled_keys": None, "records": None, "seconds": None, diff --git a/buildkite/tests/pipeline_generator/test_global_config.py b/buildkite/tests/pipeline_generator/test_global_config.py index 84398d863..19e4b18ae 100644 --- a/buildkite/tests/pipeline_generator/test_global_config.py +++ b/buildkite/tests/pipeline_generator/test_global_config.py @@ -220,3 +220,10 @@ def test_amd_run_all_patterns_use_the_shared_excludes(): assert not match(["csrc/rocm/a.cu"], patterns, excludes) assert not match(["vllm/a.py"], patterns, excludes) assert not match(["csrc/a.cu"], None, excludes), "no patterns, no match" + + +def test_parse_step_keys_names_the_include_variable(): + from buildkite.pipeline_generator.global_config import INCLUDE_STEP_KEYS_ENV_VAR + + with pytest.raises(ValueError, match=INCLUDE_STEP_KEYS_ENV_VAR): + _parse_step_keys("[]", INCLUDE_STEP_KEYS_ENV_VAR) diff --git a/buildkite/tests/pipeline_generator/test_selector_gate.py b/buildkite/tests/pipeline_generator/test_selector_gate.py index 96fe74ede..e91f7eb4b 100644 --- a/buildkite/tests/pipeline_generator/test_selector_gate.py +++ b/buildkite/tests/pipeline_generator/test_selector_gate.py @@ -6,6 +6,7 @@ import subprocess import pytest +import yaml import selector_gate from constants import Platform @@ -437,3 +438,97 @@ def emission(**kw): assert record["selector_changed_steps"] == ["nightly-changed"] assert record["selector_keys"] == ["nightly-changed", "test-b"] assert record["selector_optional_dropped"] == ["nightly-other"] + + +# `/ci run --include ...` (VLLM_CI_INCLUDE_STEP_KEYS) + + +def _steps_with_optional(): + return _steps() + [ + Step( + label="Opt", + key="test-opt", + depends_on=["image-build"], + commands=["opt"], + source_file_dependencies=["vllm/"], + optional=True, + ), + ] + + +def _generate(config, monkeypatch, tmp_path, steps=_steps_with_optional): + monkeypatch.setattr(pg, "read_steps_from_job_dir", lambda _: steps()) + monkeypatch.delenv("BUILDKITE_JOB_ID", raising=False) + monkeypatch.setenv("PATH", "") # no buildkite-agent: uploads are skipped + annotations = [] + monkeypatch.setattr( + pg, "_annotate_best_effort", lambda msg, style=None: annotations.append(msg) + ) + config["job_dirs"] = ["jobs"] + generator = object.__new__(pg.PipelineGenerator) + generator.output_file_path = str(tmp_path / "pipeline.yaml") + generator.generate() + return annotations + + +def _ready_keys(tmp_path): + """Keys that run without anyone unblocking them.""" + pipeline = yaml.safe_load(open(tmp_path / "pipeline.yaml")) + return { + step.get("key") + for group in pipeline["steps"] + for step in group.get("steps", [group]) + if step.get("key") + } + + +def test_requested_jobs_run_beside_the_selectors(pr_config, monkeypatch, tmp_path): + """An included optional step runs with the selector's choice, and is + recorded apart from today's rules, the baseline the selector is scored + against.""" + monkeypatch.setattr(selector_gate, "run_ci_select", _emission(["test-b"])) + pr_config["include_step_keys"] = frozenset({"test-opt"}) + _generate(pr_config, monkeypatch, tmp_path) + + record = json.load(open(tmp_path / selector_gate.DECISION_FILE)) + assert record["applied"] is True + assert record["legacy_keys"] == ["image-build", "test-a"] + assert record["included_keys"] == ["image-build", "test-opt"] + assert record["scheduled_keys"] == ["image-build", "test-b", "test-opt"] + assert {"test-b", "test-opt"} <= _ready_keys(tmp_path) + assert "test-a" not in _ready_keys(tmp_path) + + +def test_requested_jobs_stay_out_of_todays_rules(pr_config): + pr_config["include_step_keys"] = frozenset({"test-opt", "test-b"}) + assert pg._legacy_keys(_steps_with_optional(), ["vllm/foo.py"]) == [ + "image-build", + "test-a", + ] + assert pr_config["include_step_keys"] == frozenset({"test-opt", "test-b"}) + + +def test_an_unknown_requested_key_keeps_normal_ci(pr_config, monkeypatch, tmp_path): + monkeypatch.setattr(selector_gate, "run_ci_select", _emission(["test-b"])) + pr_config["include_step_keys"] = frozenset({"no-such-step"}) + annotations = _generate(pr_config, monkeypatch, tmp_path) + + assert any("Ignoring requested extra jobs" in a for a in annotations) + record = json.load(open(tmp_path / selector_gate.DECISION_FILE)) + assert record["included_keys"] is None + assert record["scheduled_keys"] == ["image-build", "test-b"] + assert "test-opt" not in _ready_keys(tmp_path) + + +def test_requested_jobs_run_on_a_docs_only_change(pr_config, monkeypatch, tmp_path): + pr_config.update(docs_only_disable="0", list_file_diff=["docs/index.md"]) + pr_config["include_step_keys"] = frozenset({"test-opt"}) + _generate(pr_config, monkeypatch, tmp_path) + assert "test-opt" in _ready_keys(tmp_path) + + +def test_include_cannot_be_combined_with_only(pr_config, monkeypatch, tmp_path): + pr_config["include_step_keys"] = frozenset({"test-opt"}) + pr_config["only_step_keys"] = frozenset({"test-a"}) + with pytest.raises(ValueError, match="cannot be combined"): + _generate(pr_config, monkeypatch, tmp_path) diff --git a/buildkite/tests/pipeline_generator/test_step.py b/buildkite/tests/pipeline_generator/test_step.py index 92c119933..0bb18a189 100644 --- a/buildkite/tests/pipeline_generator/test_step.py +++ b/buildkite/tests/pipeline_generator/test_step.py @@ -1211,3 +1211,23 @@ def test_docker_step_queue_routing(monkeypatch, key, label, branch, queue): monkeypatch.setattr(buildkite_step, "get_global_config", lambda: config) step = Step(label=label, key=key, commands=["true"]) assert buildkite_step.get_agent_queue(step) == queue + + +def test_an_included_optional_step_runs_without_source_match(fake_global_config): + fake_global_config["include_step_keys"] = frozenset({"extra"}) + extra = Step(label="Extra", key="extra", optional=True, commands=["t"]) + other = Step( + label="Other", + key="other", + source_file_dependencies=["unrelated.py"], + commands=["t"], + ) + assert buildkite_step._step_should_run(extra, ["changed.py"]) + assert not buildkite_step._step_should_run(other, ["changed.py"]) + + +def test_noauto_still_defers_included_steps(fake_global_config, monkeypatch): + monkeypatch.setenv("NOAUTO", "1") + fake_global_config["include_step_keys"] = frozenset({"extra"}) + extra = Step(label="Extra", key="extra", commands=["t"]) + assert not buildkite_step._step_should_run(extra, ["changed.py"])