diff --git a/buildkite/pipeline_generator/buildkite_step.py b/buildkite/pipeline_generator/buildkite_step.py index f6c7daa19..30fcebecb 100644 --- a/buildkite/pipeline_generator/buildkite_step.py +++ b/buildkite/pipeline_generator/buildkite_step.py @@ -120,10 +120,13 @@ def _otel_setup_command() -> str: 'ci_otel_run() { shift 2; env "$$@"; return $$?; }; ' 'CI_INFRA_OTEL_DIR="$${CI_INFRA_OTEL_DIR:-' # Native AMD pods run from /vllm-workspace, which has no .git; their - # checkout is BUILDKITE_BUILD_CHECKOUT_PATH. The fallback keeps the + # checkout is BUILDKITE_BUILD_CHECKOUT_PATH. The CPU test image has no + # .git either and the docker plugin passes no checkout path in, so the + # last resort is the image's own copy. The fallback keeps the # assignment successful under `sh -e`; a missing helper stays fail-open. "$$(git rev-parse --show-toplevel 2>/dev/null || " - 'printf %s "$${BUILDKITE_BUILD_CHECKOUT_PATH:-}")/' + 'printf %s "$${BUILDKITE_BUILD_CHECKOUT_PATH:-' + f'{runtime_shard.CONTAINER_WORKSPACE}}}")/' '.buildkite/scripts/ci-otel}"; export CI_INFRA_OTEL_DIR; ' 'if [ -f "$$CI_INFRA_OTEL_DIR/ci_otel.sh" ] && ' 'sh -n "$$CI_INFRA_OTEL_DIR/ci_otel.sh" && ' diff --git a/buildkite/tests/pipeline_generator/test_step.py b/buildkite/tests/pipeline_generator/test_step.py index 952ebfbf9..c31b07c2c 100644 --- a/buildkite/tests/pipeline_generator/test_step.py +++ b/buildkite/tests/pipeline_generator/test_step.py @@ -375,7 +375,7 @@ def test_otel_trace_wraps_every_command_on_trusted_main(fake_global_config): assert ".buildkite/scripts/ci-otel" in commands[0] assert "base64 --decode" not in commands[0] - assert len(commands[0]) < 500 + assert len(commands[0]) < 520 assert '. "$$CI_INFRA_OTEL_DIR/ci_otel.sh"' in commands[0] assert "ci_otel_start 1 " in commands[2] assert "export VALUE=ready" in commands[2] @@ -709,6 +709,39 @@ def test_otel_setup_finds_helpers_in_checkout_outside_git(tmp_path): assert "helper-loaded" in result.stderr +def test_otel_setup_finds_helpers_in_image_workspace(tmp_path): + # The CPU test image has no .git, and the docker plugin doesn't pass the + # checkout path in: the helpers are only in the image's own workspace. + workspace = tmp_path / "vllm-workspace" + helper_dir = workspace / ".buildkite" / "scripts" / "ci-otel" + helper_dir.mkdir(parents=True) + (helper_dir / "ci_otel.sh").write_text("echo helper-loaded >&2\n") + tests_dir = workspace / "tests" + tests_dir.mkdir() + env = { + k: v + for k, v in os.environ.items() + if k not in ("CI_INFRA_OTEL_DIR", "BUILDKITE_BUILD_CHECKOUT_PATH") + } + env["GIT_CEILING_DIRECTORIES"] = str(tmp_path) + script = buildkite_step._otel_setup_command().replace("$$", "$") + assert script.count("/vllm-workspace") == 1 + script = script.replace("/vllm-workspace", str(workspace)) + + result = subprocess.run( + ["/bin/sh", "-e", "-c", script], + cwd=tests_dir, + check=False, + capture_output=True, + text=True, + env=env, + ) + + assert result.returncode == 0, result.stderr + assert "helper-loaded" in result.stderr + assert "tracing setup skipped" not in result.stderr + + @pytest.mark.parametrize("dind, traced", [(False, True), (True, False)]) def test_otel_trace_covers_native_amd_mirrors_only(fake_global_config, dind, traced): fake_global_config["branch"] = "main"