From 0ee1b7d557c484a558018c6110ee58087aaa0220 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Thu, 3 Sep 2026 13:06:27 -0700 Subject: [PATCH 01/11] =?UTF-8?q?feat(cli):=20add=20`coder-eval=20execute`?= =?UTF-8?q?=20=E2=80=94=20run=20tasks=20without=20grading=20them?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `coder-eval execute` is `coder-eval run` with the grading half removed: the agent runs and the full trajectory lands in task.json as usual, but no criterion is checked, `weighted_score` stays None, and the row finalizes as the new `FinalStatus.NOT_GRADED`. It exists so an external harness can own the verdict — the motivating case is Harbor (Terminal-Bench 2.0), which builds its own container, calls coder-eval as the agent, and grades with its own tests/test.sh. Grading twice there would be worse than not grading: coder-eval's verdict would be reported alongside Harbor's without being the one that counts. ## NOT_GRADED is a fourth reporting category `FinalStatus.NOT_GRADED.category == "ungraded"`, not a fold into one of the existing three — folding into "failed" would depress every pass rate, into "succeeded" would invent verdicts, into "error" would report a healthy run as broken. Ungraded rows therefore leave BOTH sides of every rate: `RunSummary` / `VariantAggregate` `pass_rate` and `error_share` now divide by `tasks_graded` (`tasks_run - tasks_not_graded`), which is identical to `tasks_run` for every graded run. `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter, and is defaulted so pre-existing run.json/experiment.json still parse. `weighted_score` is set to None explicitly rather than left to `calculate_weighted_score`, which writes 0.0 for an empty results list — a value indistinguishable from "graded and scored zero" that every downstream `score or 0.0` would launder into a real-looking failure. Only SUCCESS/FAILURE collapse into NOT_GRADED. ERROR, TIMEOUT, BUILD_FAILED, MAX_TURNS_EXHAUSTED and the budget stops are facts about the *run*, not about grading, so they still apply and `execute` still exits non-zero on a crash. ## The switch `BatchRunConfig.grade` -> `Orchestrator(grade=...)`, gating all four grading call sites (single-shot, evaluate-only, the simulation dialog check, post-failure diagnostics). It crosses the docker boundary in context.json, defaulting to True in-container so a host predating `execute` keeps grading. It is deliberately NOT a task-config field: no 5-layer merge, no -D path. A task YAML must never be able to declare itself ungraded; only the invoking command decides. `run` and `execute` share one body (`run_command.run_pipeline`) and differ solely in that flag — no third code path. Only the Typer signature is restated, and a test asserts the two option sets stay in step. ## Refused rather than degraded - `--junit-xml`: a report of verdicts, and there are none. (reports_junit still emits for an ungraded row it encounters elsewhere.) - `--resume`: partition_for_resume treats "has any final status" as finalized, so a NOT_GRADED row would be skipped by a later `run --resume` rather than graded. - Simulation tasks: the dialog loop reads criteria results to decide whether to keep talking, so an ungraded dialog would silently change its own stopping behavior. Rejected by name at startup. - `stop_early:` blocks go inert: early stop cuts a run once the criteria decide the outcome, and here the full trajectory is the deliverable. ## Ripple The explicit-mapping guards did their job — every surface below failed loudly rather than silently mis-bucketing the new member: pyright on `reports_junit._category_of`, the `_status_badge` category tests, the published-action gate's "every FinalStatus must be classified" test, and CE018's enum-parity check. - reports_junit: ungraded -> (already counted by _set_counts). - reports_html: neutral badge; the "no member falls through to neutral" guard now allows it for ungraded only. - reports / reports_experiment: a "Not Graded" line, and the pass rate reads "n/a" for a fully ungraded run instead of 0.0% (an ordinary EMPTY run keeps its original 0/0 rendering — different facts). - experiment aggregation: average_score means over graded rows only, and _pick_worst_status ranks ungraded least-urgent so any real verdict wins. - verify-published-action.yml: NOT_GRADED hard-fails. That job runs the published action, which always grades, so reaching it means the action is dispatching the wrong command and every score gate is measuring nothing. - evalboard statusCategory: NOT_GRADED -> "unknown", the category every consumer already treats as "no verdict here". Not a pass, not a failure. ## Verification `make verify` and `make evalboard-verify` both green. The new suite covers the status semantics, an end-to-end execute against the agentless task (asserting pre_run's file IS written, so a skipped task can't pass), a negative control proving `run` still scores that same task 1.0, the docker context.json round-trip, and the run/execute signature parity. Scoped out of this PR: relaxing the non-empty `success_criteria` validator. `execute` on an existing task YAML needs no such change; it is only needed for a foreign task format that has no criteria to declare, and belongs with that work. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/verify-published-action.yml | 12 + CLAUDE.md | 4 +- docs/REPORT_SCHEMA.md | 21 +- docs/USER_GUIDE.md | 34 ++ evalboard/lib/status.ts | 7 + src/coder_eval/cli/__init__.py | 5 +- src/coder_eval/cli/execute_command.py | 224 +++++++++++++ src/coder_eval/cli/run_command.py | 90 +++++- src/coder_eval/cli/run_helpers.py | 9 +- .../cli/run_task_internal_command.py | 5 + src/coder_eval/isolation/docker_runner.py | 8 + src/coder_eval/models/enums.py | 18 +- src/coder_eval/models/experiment.py | 24 +- src/coder_eval/models/results.py | 37 ++- src/coder_eval/orchestration/batch.py | 3 + src/coder_eval/orchestration/config.py | 14 + src/coder_eval/orchestration/experiment.py | 20 +- src/coder_eval/orchestrator.py | 75 ++++- src/coder_eval/reports.py | 13 +- src/coder_eval/reports_experiment.py | 5 +- src/coder_eval/reports_html.py | 5 +- src/coder_eval/reports_junit.py | 10 +- .../ce018_no_final_status_name_denylist.py | 1 + tests/test_execute_command.py | 293 ++++++++++++++++++ tests/test_reports_html.py | 19 +- 25 files changed, 920 insertions(+), 36 deletions(-) create mode 100644 src/coder_eval/cli/execute_command.py create mode 100644 tests/test_execute_command.py diff --git a/.github/workflows/verify-published-action.yml b/.github/workflows/verify-published-action.yml index 134dbeb1..914cbdc0 100644 --- a/.github/workflows/verify-published-action.yml +++ b/.github/workflows/verify-published-action.yml @@ -541,6 +541,18 @@ jobs: "uploaded run dir before re-running; this is an unattended paid job.") sys.exit(1) + # NOT_GRADED means a task ran but was never scored. This job invokes the + # published action, which runs `coder-eval run` (graded), so reaching it is + # impossible unless the action started dispatching `coder-eval execute` -- + # in which case every score gate below silently measures nothing and the job + # goes green having verified no verdict at all. Hard-fail, don't tolerate. + ungraded = [s for s in statuses if s == "NOT_GRADED"] + if ungraded: + print("::error::task(s) reported NOT_GRADED -- the published action ran without " + "grading. `coder-eval run` always grades, so the action is dispatching the " + "wrong command and every score gate in this job is measuring nothing.") + sys.exit(1) + # Exit-contract check, conditional on the model having actually performed. # Ignoring the step's exit code entirely (see the continue-on-error rationale # above) would also hide a REGRESSION in the action's own exit logic -- e.g. a diff --git a/CLAUDE.md b/CLAUDE.md index ac00d5ef..896c2e4b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -89,7 +89,8 @@ coder_eval/ │ ├── cli/ # CLI commands (Typer + Rich) │ ├── __init__.py # Typer app setup (core commands) -│ ├── run_command.py # `coder-eval run` +│ ├── run_command.py # `coder-eval run` + `run_pipeline` (the body BOTH run and execute share) +│ ├── execute_command.py # `coder-eval execute` — Typer signature only; delegates to run_pipeline(grade=False) │ ├── plan_command.py # `coder-eval plan` │ ├── report_command.py # `coder-eval report` │ ├── run_helpers.py # CLI helper functions @@ -146,6 +147,7 @@ action.yml # Published composite GitHub Action (coder-ev - **Reference solutions are directory-only, and shielded (partially) from the agent**: `task.reference` is a single required `directory:` (relative to the task YAML) — the inline `code:` / single-file `file:` forms are gone, because a directory is the only shape that can be permission-gated as a unit; a `model_validator(mode="before")` gives the removed forms a migration error. The orchestrator stages a **per-run private copy** (`orchestration/evaluation.py::stage_reference_dir`, symlinks stripped) into a tempdir, removed in `_cleanup` via `path_utils.rmtree_restrictive` (keyed on `_reference_staging_root`, recorded BEFORE the copy so a failed copy still cleans up; `rmtree(ignore_errors=True)` silently declines on a tree left at 000) and deliberately never preserved into `run_dir/artifacts`. That copy is held at mode `000` for the whole of every `agent.communicate` call via **`Sandbox.set_permissions`**, the driver-aware wrapper over `fs_permissions.py::set_permissions`. Windows **stack**: exiting restores the *enclosing* window's mode, only the outermost exit restores the pre-window mode — that is what makes a mid-turn re-grant (`mode=READ_ONLY_MODE`) expressible, and it covers two windows at the same mode so no refcount is needed. The window is enforced **only inside a docker container** (`Sandbox.enforces_permission_windows`) and is a no-op on the host, where the agent shares our uid. **That gate keys on the `CODER_EVAL_IN_CONTAINER` env var, NOT `sandbox.driver`** — `run_task_internal_command` rewrites `driver: docker` → `tempdir` before building the in-container Orchestrator, so a driver-based gate would silently disable the anti-cheat on exactly the path that needs it (regression-guarded by `TestSandboxDriverGate`); `resolve_reference_dir` gates its `/work/references` branch on the same var for the same reason. The task directory is **not** shielded (`:ro` mount → EROFS, and the same YAML is readable at `/work/input`). Criteria address reference files with the `$REFERENCE_DIR` token (same resolver as `$TASK_DIR`) and the `REFERENCE_DIR` env var for `run_command`; `reference_comparison` names one file via `reference_file`. Docker mounts a throwaway **read-write** copy at `/work/references` (a `:ro` mount cannot be chmod'd — EROFS), masks the in-task-dir original with an empty tmpfs, and drops `DAC_OVERRIDE`/`DAC_READ_SEARCH`. `FOWNER`/`CHOWN` are deliberately **NOT** dropped: the in-container orchestrator that applies the window is the same root process with the same caps, so dropping `FOWNER` breaks *the harness's own* chmod wherever the bind mount preserves a non-root owner (native Linux — verified: `chmod: Operation not permitted`), i.e. exactly where the drop would otherwise bite. A window that cannot be applied is now a hard error, not a warning: `Sandbox.set_permissions` passes `strict=True` whenever it enforces, so an unprotected run fails instead of producing a normal-looking score. **KNOWN GAP — this is defense-in-depth, not a boundary**: (a) `chmod(2)` is gated on owner-or-`CAP_FOWNER` and the container runs as root owning the copy, so a deliberate `chmod 755 /work/references` restores access; (b) the window spans `agent.communicate` only, and nothing reaps agent child processes at turn end, so a backgrounded read loop succeeds once the window closes. The **write** half of (b) is closed — `path_utils.digest_tree` hashes the tree at staging and `Orchestrator._verify_reference_integrity` re-checks before grading, raising `ReferenceTamperedError` (→ `FinalStatus.ERROR`) on a mismatch so an agent cannot overwrite the reference to drive `reference_comparison` to 1.0. Passive reads are blocked; an adversarial agent is not. Full containment requires running the agent as a non-root uid AND holding the window for the agent's whole lifetime — follow-up. `tasks/anti_cheat_reference` probes the passive-read half. - **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex and Antigravity (both run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced), and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. Full table + rationale: docs/agents/HARNESS_PARITY.md. - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly. +- **Execute vs. run (the grading switch)**: `coder-eval execute` is `coder-eval run` with grading removed — the agent runs and the full trajectory is captured, but no criterion is checked, `weighted_score` is `None` (never `0.0`, which would be indistinguishable from "graded and scored zero"), and the row finalizes as **`FinalStatus.NOT_GRADED`**, whose `category` is a **fourth** bucket, `"ungraded"`. Ungraded rows leave BOTH sides of every rate: `RunSummary.pass_rate` / `error_share` and `VariantAggregate.pass_rate` divide by `tasks_graded` (`tasks_run - tasks_not_graded`), and `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter. **Only SUCCESS/FAILURE collapse into it** — `ERROR`, `TIMEOUT`, `BUILD_FAILED`, `MAX_TURNS_EXHAUSTED` and the budget stops are facts about the *run*, not about grading, and still apply (so `execute` still exits non-zero on a crash). The switch is `BatchRunConfig.grade` → `Orchestrator(grade=...)` → the **four** grading call sites (single-shot, evaluate-only, the simulation dialog check, and post-failure diagnostics); it crosses the docker boundary in `context.json` (defaulting to `True` in-container, so a host predating `execute` keeps grading). It is **deliberately not a task-config field** — no 5-layer merge, no `-D` path — because a task YAML must never declare itself ungraded; only the invoking command decides. `run` and `execute` share one body (`run_command.run_pipeline`) and differ solely in that flag, so there is no third code path. Three things are refused rather than degraded: `--junit-xml` (a report of verdicts, and there are none — though `reports_junit` still emits `` for an ungraded row it encounters), `--resume` (`partition_for_resume` treats "has any final status" as finalized, so a `NOT_GRADED` row would be *skipped* by a later `run --resume` instead of graded), and simulation tasks (their turn-continuation logic reads criteria results, so an ungraded dialog would silently change its own stopping behavior). `stop_early:` blocks are inert under `execute` for the same reason the kill switch exists: the full trajectory is the deliverable. Motivating consumer: an external harness (Harbor / Terminal-Bench 2.0) that builds its own container, calls coder-eval as the agent, and grades with its own tests. - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `stop_early.decide_within`, deliberately lives on `LiveSuccessCriterion` instead (see below) — the watcher must attribute a decision-step timeout to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express. - **Early stop on criterion (opt-in, per-criterion arming)**: a `stop_early:` block (`StopEarlyPolicy`) on a criterion ends a single-shot run early once the run's **armed** criteria decide the outcome, so a raised `max_turns` isn't wasted on the smoke flavor. The block's PRESENCE is the arming and alone activates the watcher — there is **no run-level master switch**: `run_limits.stop_early: false` is the run-level KILL SWITCH that force-disarms every block (the one-line experiment-variant/`-D` override for an authoritative full run), and `run_limits.stop_early: true` (the removed master arm) is a hard `EarlyStopConfigError` at resolution. The block exists on `LiveSuccessCriterion` only (currently `skill_triggered`, `command_executed` — so arming an unobservable criterion is unrepresentable, a pydantic extra-forbid error). Arming carries one implicit trigger (a native live-fail may fail-stop the run); its keys refine it: `on_pass: stop` (pass-stop the moment the criterion live-passes; default `continue` just latches) and `decide_within: N` (still undecided after N tool-call steps latches an **effective fail**, fed through the same fail-stop rule, reported as `decision_budget_exceeded` — an ordinary weighted fail, NOT a gate-bypassing force-fail; cumulative across retry attempts of the same turn). A trigger whose polarity the instance can't decide (per the abstract, checker-independent `live_decidable_polarities()`, a pure function of the criterion's own fields, paired with the checker's `live_verdict` override by lint rule CE025, a registry-based whole-tree check) is **inert by design** — one dataset-fanned YAML line serves both positive rows (pass/timeout live) and distractor rows (fail live). Verdicts **latch**: once a criterion decides, its `live_verdict` is never polled again. Stop rule is weighted, not strict-boolean: `run_limits.stop_early_gate_threshold` (default `1.0`, reproducing strict-AND behavior exactly) is the minimum weighted score (`Σ weight·score / Σ weight` over the armed subset) required to pass; a fail-stop fires once the armed set's **ceiling** (best case for everything still undecided) can no longer reach the threshold — so a low-weight fail or timeout that can't doom the gate is absorbed and the run continues — and is **deferred while any pass-capable armed criterion is undecided** (a distractor misfire never truncates a positive row's recall signal); a pass-stop fires once the `on_pass: stop` subset's **floor** (worst case) already meets the threshold, and is symmetrically **deferred while any pass-capable armed criterion outside the `on_pass: stop` subset is undecided** (so an early pass never freezes a sibling `on_pass: continue` criterion's signal out of the trajectory). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a kill-switched (`stop_early: false`) run. Driven by `orchestration/early_stop.py::EarlyStopWatcher` (built when `early_stop_active(task)`: ≥1 armed criterion, kill switch not thrown) through the agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. Gating is **FIRED-ONLY**: a run the watcher actually cut gates on the **armed subset** via the weighted `EvaluationResult.armed_criteria_passed`; a run that completes naturally — armed or not — gates strict-AND via `all_criteria_passed`, so adding a block never changes the verdict of a run it didn't cut. Note the gate keys on the watcher having FIRED (`result.early_stop is not None`), not on confirmed truncation — an agent that ignores `should_stop`, or a stop firing on the final message, still gates armed-only. Every resolution-time guardrail violation is a hard error at resolution (plan *and* run); the one load-time case — a `stop_early:` block on a non-live criterion — is a pydantic schema error at task load, which the run surface reports as a skipped task like any other malformed task. A runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo` (incl. `gate_threshold` at stop time), report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. No blocks anywhere ⇒ behavior byte-for-byte unchanged. diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index f9d5554a..b4c5dce0 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -44,7 +44,8 @@ run-level summary; full per-replicate detail lives in each `task.json`. | `start_time` / `end_time` | `datetime` | Run window. | | `total_duration_seconds` | `float` | Wall-clock. | | `tasks_run` | `int` | Total replicates executed. | -| `tasks_succeeded` / `tasks_failed` / `tasks_error` | `int` | Category counts. **Invariant:** the three sum to `tasks_run`. | +| `tasks_succeeded` / `tasks_failed` / `tasks_error` / `tasks_not_graded` | `int` | Category counts. **Invariant:** the four sum to `tasks_run`. | +| `tasks_not_graded` | `int` | Tasks run by `coder-eval execute` — executed, deliberately unscored. Excluded from **both** sides of `pass_rate`. Defaults to `0`, so pre-`execute` `run.json` still parses. | | `tasks_token_budget_exceeded` / `tasks_cost_budget_exceeded` | `int` | Sub-counters of `tasks_failed` (not part of the invariant). | | `skipped_tasks` | `list[{path, reason}]` | Load failures / `skip: true` opt-outs. | | `max_parallel` | `int` | Concurrency used. | @@ -59,8 +60,9 @@ publishing different numbers for the same run. | Key | Type | Meaning | | --- | --- | --- | -| `pass_rate` | `float \| None` | `tasks_succeeded / tasks_run` — errors are in the denominator, counted as misses. `None` on an empty run (0/0 is unknown, not 0%). | -| `error_share` | `float \| None` | `tasks_error / tasks_run`. Diagnostic only; never adjusts the rate. | +| `pass_rate` | `float \| None` | `tasks_succeeded / tasks_graded` — errors are in the denominator, counted as misses; ungraded tasks are in neither. `None` on an empty or fully ungraded run (0/0 is unknown, not 0%). | +| `error_share` | `float \| None` | `tasks_error / tasks_graded`. Diagnostic only; never adjusts the rate. | +| `tasks_graded` | `int` | `tasks_run - tasks_not_graded`. The denominator of both rates above. | | `total_cost_usd` | `float \| None` | **The bill**: agent + judge + simulator, summed over the rows. `None` when nothing could be priced. | | `agent_cost_usd` | `float \| None` | Subject-agent spend alone. The harness-vs-harness comparison figure — judge spend is a property of the suite's criteria and identical across harnesses, so leaving it in would make two harnesses look closer than they are. | | `eval_overhead_cost_usd` | `float \| None` | Judge + simulator spend. The other half of `total_cost_usd`. | @@ -232,7 +234,8 @@ the same weighted armed gate as a native fail), ## `variant.json` — `VariantAggregate` A single aggregate (not wrapped): `variant_id`, `tasks_run`, `tasks_succeeded`, -`tasks_failed`, `tasks_error` (same sum-to-`tasks_run` invariant), `average_score`, +`tasks_failed`, `tasks_error`, `tasks_not_graded` (same sum-to-`tasks_run` invariant), `average_score` +(the mean over **graded** rows only), `average_duration`, `total_tokens`, `replicate_count`, `tasks_token_budget_exceeded`, `tasks_cost_budget_exceeded`. @@ -308,10 +311,20 @@ String enum values and their reporting category: | `COST_BUDGET_EXCEEDED` | failed | `$` | | `ERROR` | error | `!` | | `BUILD_FAILED` | error | `B` | +| `NOT_GRADED` | ungraded | `?` | > **Gotcha:** `BUILD_FAILED` (a failed Docker image build) categorizes as **error**, > not failed — easy to miscount downstream. +`NOT_GRADED` is produced only by [`coder-eval execute`](USER_GUIDE.md#coder-eval-execute--run-without-grading): +the task ran and its full trajectory was captured, but no criterion was checked, so +`weighted_score` is `None` (**not** `0.0` — that would be indistinguishable from a task +that was graded and scored zero). `ungraded` is a fourth reporting category, not a fold +into one of the other three: counting it as failed would depress every pass rate, and +counting it as succeeded would invent a verdict. Execution facts still win over it — a +crash, timeout, or budget breach under `execute` reports `ERROR` / `TIMEOUT` / +`TOKEN_BUDGET_EXCEEDED` as usual. + `TOKEN_BUDGET_EXCEEDED` and `COST_BUDGET_EXCEEDED` are produced by the cumulative budget caps under `run_limits:` (`max_input_tokens` / `max_output_tokens` / `max_total_tokens`, and `max_usd` respectively), checked after each completed agent turn — see diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 1676690d..878e72a9 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -58,6 +58,40 @@ flags of their own. They live under `run_limits:` in the task YAML, or on the co `-D run_limits.=` (e.g. `-D run_limits.max_usd=2.50`). The complete field reference is in the [Task Definition Guide](TASK_DEFINITION_GUIDE.md#run-limits). +### `coder-eval execute` — run without grading + +```bash +coder-eval execute tasks/hello_date.yaml # run, capture, score nothing +coder-eval execute tasks/*.yaml --run-dir ./my-run -j 3 # every `run` flag but two +``` + +Identical to `coder-eval run` except that no success criterion is checked. Each task +executes normally and its full trajectory lands in `task.json` as usual, but +`weighted_score` stays `null` and the row finalizes as `NOT_GRADED` — a reporting +category of its own, excluded from both sides of every pass rate. The two commands +share one implementation, so they cannot drift apart. + +Use it when something *else* owns the verdict — an external harness that builds its own +container and runs its own tests — or to separate one expensive agent run from grading +you want to iterate on afterwards. Grade the results later with +[`coder-eval evaluate`](#coder-eval-evaluate--test-criteria-without-an-agent). + +**Only the verdict is withheld, never the facts of the run.** A crash, timeout, or +budget breach still reports `ERROR` / `TIMEOUT` / `TOKEN_BUDGET_EXCEEDED` and still +exits non-zero, exactly as under `run`. + +Every `run` flag is available except three things, each refused rather than quietly +degraded: + +| Not supported | Why | +| --- | --- | +| `--junit-xml` | A JUnit report reports verdicts, and there are none. | +| `--resume` | Resume treats "has any final status" as finalized, so a `NOT_GRADED` row would be *skipped* by a later `run --resume` rather than graded. | +| Simulation tasks | The dialog loop reads criteria results to decide whether to keep talking, so an ungraded dialog would silently change its own stopping behavior. Rejected by name at startup. | + +`stop_early:` blocks are also inert here: early stop exists to cut a run once the +criteria decide the outcome, and under `execute` the full trajectory is the deliverable. + ### `coder-eval plan` — validate tasks ```bash diff --git a/evalboard/lib/status.ts b/evalboard/lib/status.ts index f2b02520..17796f4c 100644 --- a/evalboard/lib/status.ts +++ b/evalboard/lib/status.ts @@ -2,8 +2,14 @@ // Mirrors coder_eval `FinalStatus.category` (src/coder_eval/models/enums.py): // SUCCESS -> passed // ERROR / BUILD_FAILED -> error (BUILD_FAILED is an environment/setup failure) +// NOT_GRADED -> unknown (`coder-eval execute`: ran, deliberately unscored) // anything else (FAILURE, TIMEOUT, MAX_TURNS_EXHAUSTED, …) -> failed // +// NOT_GRADED maps to "unknown" rather than gaining a category of its own: every +// consumer already handles "unknown" (a null status) as "no verdict here", which +// is exactly what an ungraded row is. It is therefore not a pass, not a failure, +// and sorts in the middle — the same treatment a missing status gets. +// // Note: this only categorizes coder_eval task statuses. UI status display // (e.g. StatusPill) also handles flow execution statuses like "Completed" // and "Faulted" and uses its own logic. @@ -16,6 +22,7 @@ export function statusCategory(status: string | null): StatusCategory { if (!status) return "unknown"; if (status === "SUCCESS") return "passed"; if (status === "ERROR" || status === "BUILD_FAILED") return "error"; + if (status === "NOT_GRADED") return "unknown"; return "failed"; } diff --git a/src/coder_eval/cli/__init__.py b/src/coder_eval/cli/__init__.py index 40eab01d..0ded1cae 100644 --- a/src/coder_eval/cli/__init__.py +++ b/src/coder_eval/cli/__init__.py @@ -7,6 +7,7 @@ from .aggregate_command import aggregate_command from .console import console from .evaluate_command import evaluate_command +from .execute_command import execute_command from .plan_command import plan_command from .report_command import report_command from .run_command import run_command @@ -46,7 +47,8 @@ def main( Run 'coder-eval COMMAND --help' for help on a specific command. Available commands: - - run: Execute evaluation tasks + - run: Execute evaluation tasks and grade them + - execute: Execute evaluation tasks WITHOUT grading them - plan: Validate task files (dry-run) - evaluate: Run criteria against a directory without an agent - report: Display or export evaluation reports @@ -75,6 +77,7 @@ def main( # emits a CoderEval.Cli. event (Status/DurationMs/ErrorType) on completion; # functools.wraps preserves the signature so Typer still parses each command's flags. app.command(name="run")(track_command("run")(run_command)) +app.command(name="execute")(track_command("execute")(execute_command)) app.command(name="plan")(track_command("plan")(plan_command)) app.command(name="evaluate")(track_command("evaluate")(evaluate_command)) app.command(name="report")(track_command("report")(report_command)) diff --git a/src/coder_eval/cli/execute_command.py b/src/coder_eval/cli/execute_command.py new file mode 100644 index 00000000..b90fa0e3 --- /dev/null +++ b/src/coder_eval/cli/execute_command.py @@ -0,0 +1,224 @@ +"""Execute command - run evaluation tasks WITHOUT grading them. + +``coder-eval execute`` is ``coder-eval run`` with the grading half removed: the +sandbox is built, the agent runs, and the full trajectory is captured into the +usual ``task.json`` / ``run.json`` layout — but no success criterion is checked, +``weighted_score`` stays ``None``, and each row finalizes as +``FinalStatus.NOT_GRADED``. + +It exists so an *external* harness can own the verdict. The motivating case is +Harbor (Terminal-Bench 2.0), which builds its own container, calls coder-eval as +the agent, and grades with its own ``tests/test.sh``. Grading twice there would +be worse than not grading at all: coder-eval's verdict would be reported +alongside Harbor's without being the one that counts. + +Every flag on ``run`` is available here except two, and both omissions are +deliberate: + +* ``--junit-xml`` — a JUnit report is a report of verdicts, and there are none. +* ``--resume`` — ``partition_for_resume`` treats "has any final status" as + finalized, so a ``NOT_GRADED`` row would be skipped by a later ``run --resume`` + rather than graded. Supporting it needs resume to distinguish "done" from + "executed but unscored"; until then, refusing is the honest option. + +The command shares ``run``'s entire body (``run_command.run_pipeline``); only the +Typer signature is restated, because Typer builds its parser from the signature. +``tests/test_execute_command.py`` asserts the two signatures stay in step. +""" + +from pathlib import Path + +import click +import typer + +from ..models import PreservationMode +from .run_command import run_pipeline + + +def execute_command( + task_files: list[Path] | None = typer.Argument( # noqa: B008 + None, + help="Path(s) to task YAML file(s). Defaults to all tasks/ recursively.", + ), + preservation_mode: PreservationMode | None = typer.Option( # noqa: B008 + None, + "--preservation-mode", + help=( + "How to persist each task's sandbox: NONE (delete), MOVE_ON_WRITE " + "(run in a tempdir, move into run_dir/artifacts), or DIRECT_WRITE " + "(run directly in run_dir/artifacts). Default is driver-derived — " + "docker → DIRECT_WRITE, else MOVE_ON_WRITE. Explicit value always wins." + ), + ), + run_dir: Path | None = typer.Option( # noqa: B008 + None, + "--run-dir", + help="Custom run directory (default: auto-generated timestamped directory in runs/)", + ), + max_parallel: int = typer.Option( + 1, + "--max-parallel", + "-j", + help="Maximum number of tasks to run concurrently (default: 1 = sequential)", + min=1, + ), + verbose: bool = typer.Option( + False, + "--verbose", + "-v", + help="Enable verbose (DEBUG level) logging", + ), + log_file: Path | None = typer.Option( # noqa: B008 + None, + "--log-file", + help="Log to file in addition to console", + ), + tags: str | None = typer.Option( + None, + "--tags", + "-t", + help="Only run tasks matching any of these tags (comma-separated, e.g., 'smoke,golden')", + ), + exclude_tags: str | None = typer.Option( + None, + "--exclude-tags", + help="Skip tasks matching any of these tags (comma-separated, e.g., 'example,integration')", + ), + include_skipped: bool = typer.Option( + False, + "--include-skipped", + help=( + "Also run tasks marked `skip: true` in their YAML. Off by default so the " + "nightly/CI keep excluding them; use for on-demand / local runs of " + "quarantined or opt-in tasks." + ), + ), + agent_type: str | None = typer.Option( + None, + "--type", + "-T", + help="Override agent type for all tasks (e.g. 'claude-code', 'codex', or a plugin kind)", + ), + model: str | None = typer.Option( + None, + "--model", + "-m", + help="Override agent model for all tasks (e.g., claude-sonnet-4-20250514)", + ), + stream: str | None = typer.Option( + None, + "--stream", + "-s", + click_type=click.Choice(["full", "minimal"], case_sensitive=False), + help="Stream LLM events to terminal: 'full' or 'minimal' (turn-level only). Disables progress bar.", + ), + backend: str | None = typer.Option( + None, + "--backend", + "-b", + click_type=click.Choice(["direct", "bedrock", "litellm"], case_sensitive=False), + help="API backend (default: from API_BACKEND env var)", + ), + experiment: Path | None = typer.Option( # noqa: B008 + None, + "--experiment", + "-e", + help="Experiment definition YAML (default: experiments/default.yaml)", + ), + sample: int | None = typer.Option( + None, + "--sample", + help=( + "For dataset-backed tasks, use a random N-row sample " + "(fixed seed: reproducible, unbiased across paths). Cheap dataset smoke-test." + ), + min=1, + ), + sample_per_stratum: int | None = typer.Option( + None, + "--sample-per-stratum", + help=( + "For dataset-backed tasks, keep up to N rows per stratum (stratify_field, " + "default expected_skill) — a stratified sample that overrides the task's " + "dataset.sample_per_stratum without editing the YAML. Ignored when --sample is set. " + "Nondeterministic (re-draws each run) unless the task sets dataset.sample_seed." + ), + min=1, + ), + repeats: int | None = typer.Option( + None, + "--repeats", + help="Run each (task, variant) N times. Overrides experiment/variant `repeats:`. Must be >=1.", + min=1, + ), + driver: str | None = typer.Option( + None, + "--driver", + click_type=click.Choice(["tempdir", "docker"], case_sensitive=False), + help="Override sandbox driver for all tasks. 'docker' runs each task in a fresh container.", + ), + set_overrides: list[str] = typer.Option( # noqa: B008 + [], + "--set", + "-D", + metavar="PATH=VALUE", + help=( + "Override any resolved task-config field under agent/run_limits/sandbox, " + "e.g. -D run_limits.max_turns=30 -D agent.permission_mode=plan " + "-D agent.sdk_options.effort=high -D sandbox.docker.network=none. " + "Repeatable. Validated against the schema. A path set by both an alias " + "and -D is an error; values are YAML-parsed (on/off/yes/no stay strings). " + "(--model and --driver are shorthand aliases for -D agent.model / " + "-D sandbox.driver.)" + ), + ), +) -> None: + """Run evaluation tasks WITHOUT checking their success criteria. + + Identical to `coder-eval run` except that nothing is graded: each task + executes, its full trajectory is captured to task.json, and the row + finalizes as NOT_GRADED with no weighted_score. Use it when an external + harness owns the verdict, or to separate an expensive agent run from + grading you want to iterate on afterwards. + + Grade the results afterwards with `coder-eval evaluate`. + + Execution failures still fail: a crash, timeout, or budget breach reports + ERROR / TIMEOUT / TOKEN_BUDGET_EXCEEDED and exits non-zero exactly as under + `run`. Only the verdict is withheld, never the facts of the run. + + Not supported here: --junit-xml (no verdicts to report), --resume (a + NOT_GRADED row would be mistaken for a finalized one), and simulation tasks + (their turn-continuation logic reads criteria results). + + Examples: + + coder-eval execute tasks/hello_date.yaml + + coder-eval execute tasks/*.yaml --run-dir ./my-run --max-parallel 3 + """ + run_pipeline( + grade=False, + task_files=task_files, + preservation_mode=preservation_mode, + run_dir=run_dir, + # Not exposed as flags — see the module docstring for why each is refused. + resume=False, + junit_xml=None, + max_parallel=max_parallel, + verbose=verbose, + log_file=log_file, + tags=tags, + exclude_tags=exclude_tags, + include_skipped=include_skipped, + agent_type=agent_type, + model=model, + stream=stream, + backend=backend, + experiment=experiment, + sample=sample, + sample_per_stratum=sample_per_stratum, + repeats=repeats, + driver=driver, + set_overrides=set_overrides, + ) diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index 9b22929c..d820e301 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -354,6 +354,65 @@ def run_command( coder-eval run tasks/*.yaml --tags golden,basic --exclude-tags example """ + run_pipeline( + grade=True, + task_files=task_files, + preservation_mode=preservation_mode, + run_dir=run_dir, + resume=resume, + max_parallel=max_parallel, + verbose=verbose, + log_file=log_file, + junit_xml=junit_xml, + tags=tags, + exclude_tags=exclude_tags, + include_skipped=include_skipped, + agent_type=agent_type, + model=model, + stream=stream, + backend=backend, + experiment=experiment, + sample=sample, + sample_per_stratum=sample_per_stratum, + repeats=repeats, + driver=driver, + set_overrides=set_overrides, + ) + + +def run_pipeline( + *, + grade: bool, + task_files: list[Path] | None, + preservation_mode: PreservationMode | None, + run_dir: Path | None, + resume: bool, + max_parallel: int, + verbose: bool, + log_file: Path | None, + junit_xml: Path | None, + tags: str | None, + exclude_tags: str | None, + include_skipped: bool, + agent_type: str | None, + model: str | None, + stream: str | None, + backend: str | None, + experiment: Path | None, + sample: int | None, + sample_per_stratum: int | None, + repeats: int | None, + driver: str | None, + set_overrides: list[str], +) -> None: + """The shared body of ``coder-eval run`` and ``coder-eval execute``. + + Everything below the Typer signature is identical for both commands; the only + difference is ``grade``, which decides whether success criteria are checked + (``run``) or the trajectory is captured and left unscored (``execute``). Both + commands are pure flag-parsing wrappers over this function, so a behavior + change can never apply to one and miss the other. + """ # --resume needs an explicit run dir to resume into (auto-generated dirs are always fresh). if resume and run_dir is None: raise typer.BadParameter("--resume requires --run-dir pointing at the run to continue.") @@ -414,6 +473,7 @@ def run_command( resume=resume, include_skipped=include_skipped, junit_xml=junit_xml, + grade=grade, ) ) except KeyboardInterrupt: @@ -439,6 +499,7 @@ async def _run_all_tasks( resume: bool = False, include_skipped: bool = False, junit_xml: Path | None = None, + grade: bool = True, ) -> None: """Async entry point for running all tasks (optionally in parallel). @@ -459,6 +520,7 @@ async def _run_all_tasks( experiment_path: Optional path to experiment YAML (default: experiments/default.yaml) junit_xml: Optional path to write a JUnit XML report to, after the run summary is persisted and before the failure exit-code gate. + grade: False for `coder-eval execute` — run and capture, score nothing. """ # Prepare run directory run_dir = prepare_run_directory(run_dir) @@ -484,6 +546,7 @@ async def _run_all_tasks( repeats=repeats, verbose=verbose, include_skipped=include_skipped, + grade=grade, ) from ..telemetry import flush_telemetry, track_event @@ -500,6 +563,7 @@ async def _run_all_tasks( "StreamMode": stream_mode or "none", "Resume": resume, "ExperimentProvided": experiment_path is not None, + "Grade": grade, }, ) @@ -513,7 +577,7 @@ async def _run_all_tasks( try: # Always run through experiment layer (defaults to experiments/default.yaml) summary, failed_suite_gates = await _run_with_experiment( - all_task_files, config, experiment_path, stream_mode, max_parallel, resume=resume + all_task_files, config, experiment_path, stream_mode, max_parallel, resume=resume, grade=grade ) # Aggregate task logs into run.log @@ -600,6 +664,7 @@ async def _run_with_experiment( stream_mode: str | None, max_parallel: int, resume: bool = False, + grade: bool = True, ) -> tuple[RunSummary, int]: """Run tasks through the experiment resolution layer. @@ -670,6 +735,23 @@ async def _run_with_experiment( except ValueError as e: raise typer.BadParameter(str(e)) from e + # Simulation tasks are rejected under `execute`, not silently degraded. The + # dialog loop reads criteria results to decide whether to keep talking, so an + # ungraded dialog would quietly change its own stopping behavior and produce a + # trajectory that is not the one `run` would have produced. Rejecting is a + # config error (exit 2), and it names the offending tasks. + if not grade: + simulated = sorted( + rt.task.task_id for rt in resolved if rt.task.simulation is not None and rt.task.simulation.enabled + ) + if simulated: + raise typer.BadParameter( + "`coder-eval execute` does not support simulation tasks (their turn-continuation " + + "logic depends on criteria results): " + + ", ".join(simulated) + + ". Use `coder-eval run` for these." + ) + if skipped: console.print( f"[yellow]⚠[/] {len(skipped)} task file(s) skipped " @@ -745,6 +827,12 @@ async def _run_with_experiment( # Per-suite pass-rate rollups for dataset-backed tasks (no-op when none were used). # Pass `resolved` through so suite_thresholds on each criterion can be evaluated. + # Skipped entirely under `execute`: a rollup aggregates per-criterion results, + # and there are none — running it would gate a suite on an empty aggregate and + # report a threshold failure for a run that was never measured. + if not grade: + return summary, 0 + from ..reports import write_suite_rollups rollups = write_suite_rollups(config.run_dir, task_results, resolved_tasks=resolved) diff --git a/src/coder_eval/cli/run_helpers.py b/src/coder_eval/cli/run_helpers.py index 9ef304cf..90324ca2 100644 --- a/src/coder_eval/cli/run_helpers.py +++ b/src/coder_eval/cli/run_helpers.py @@ -132,7 +132,14 @@ def print_execution_summary(run_dir: Path, summary: RunSummary) -> None: summary: Run execution summary """ console.print(f"\n[bold green]Run complete:[/bold green] {run_dir}") - console.print(f"[bold]Results:[/bold] {summary.tasks_succeeded}/{summary.tasks_run} succeeded") + # An ungraded run has no pass rate to report — printing "0/N succeeded" for a + # clean `coder-eval execute` reads as a total failure. Report what actually + # happened instead, and keep the graded line for whatever WAS graded. + if summary.tasks_not_graded: + console.print(f"[bold]Results:[/bold] {summary.tasks_not_graded}/{summary.tasks_run} executed, not graded") + console.print("[dim]Grade later: uv run coder-eval evaluate [/dim]") + if summary.tasks_graded: + console.print(f"[bold]Results:[/bold] {summary.tasks_succeeded}/{summary.tasks_graded} succeeded") console.print(f"[dim]View report: open {run_dir / 'experiment.md'}[/dim]") console.print(f"[dim]View report: uv run coder-eval report {run_dir}[/dim]") diff --git a/src/coder_eval/cli/run_task_internal_command.py b/src/coder_eval/cli/run_task_internal_command.py index 3157e4a7..8bc885b5 100644 --- a/src/coder_eval/cli/run_task_internal_command.py +++ b/src/coder_eval/cli/run_task_internal_command.py @@ -153,6 +153,10 @@ def _watch_host_heartbeat() -> None: # a missing key falls back to the docker default (DIRECT_WRITE) — a deliberate # default, not version back-compat. preservation_mode = PreservationMode(context.get("preservation_mode", PreservationMode.DIRECT_WRITE.value)) + # `coder-eval run` vs `coder-eval execute`, decided host-side. Defaults to + # True (grade) so a host that predates `execute` — which never writes the + # key — keeps its exact behavior. + grade: bool = context.get("grade", True) # Docker WORKDIR alignment: the host resolves the concrete WORKDIR # (config value / "auto" -> `docker inspect` / fallback) and forwards it here. # Absent -> None -> standard run_dir/artifacts workspace. @@ -197,6 +201,7 @@ def _watch_host_heartbeat() -> None: config_lineage=config_lineage, replicate_index=replicate_index, workspace_dir=workspace_dir, + grade=grade, ) # Install the stdout-NDJSON stream callback so per-tool-call events diff --git a/src/coder_eval/isolation/docker_runner.py b/src/coder_eval/isolation/docker_runner.py index 31683d75..bb967222 100644 --- a/src/coder_eval/isolation/docker_runner.py +++ b/src/coder_eval/isolation/docker_runner.py @@ -540,11 +540,16 @@ def __init__( preservation_mode: PreservationMode = PreservationMode.DIRECT_WRITE, stream_callback: StreamCallback | None = None, verbose: bool = False, + grade: bool = True, ) -> None: self.rt = rt self.preservation_mode = preservation_mode self.stream_callback = stream_callback self.verbose = verbose + # Forwarded to the in-container orchestrator via context.json. It is a + # run-level decision made by the CLI, so it cannot be recovered from the + # staged task.yaml on the other side. + self.grade = grade # Set by _prepare_host_mounts: the tmp lean copy of ~/.claude that # _build_argv mounts read-write. None when there is no ~/.claude to # forward or the mount is opted out (CODER_EVAL_NO_CLAUDE_MOUNT). @@ -721,6 +726,9 @@ def _dump_task_yaml() -> str: "replicate_index": self.rt.replicate_index, "config_lineage": {k: v.model_dump(mode="json") for k, v in self.rt.config_lineage.items()}, "preservation_mode": self.preservation_mode.value, + # `coder-eval run` vs `coder-eval execute`. Not derivable from + # task.yaml on the container side (deliberately not a task field). + "grade": self.grade, "source_yaml": self.rt.source_yaml, # Docker WORKDIR alignment: concrete path the in-container # orchestrator runs at + captures out (None = standard workspace). diff --git a/src/coder_eval/models/enums.py b/src/coder_eval/models/enums.py index 0cba3650..7135de37 100644 --- a/src/coder_eval/models/enums.py +++ b/src/coder_eval/models/enums.py @@ -15,9 +15,16 @@ class FinalStatus(StrEnum): MAX_TURNS_EXHAUSTED = "MAX_TURNS_EXHAUSTED" TOKEN_BUDGET_EXCEEDED = "TOKEN_BUDGET_EXCEEDED" COST_BUDGET_EXCEEDED = "COST_BUDGET_EXCEEDED" + # `coder-eval execute` ran the agent but deliberately skipped grading, so + # there is no verdict to report. Distinct from FAILURE (which asserts the + # criteria were checked and did not pass) and from ERROR (which asserts + # something went wrong). Only SUCCESS/FAILURE collapse into it — every + # other member records an *execution* fact that still applies when the + # run is ungraded. + NOT_GRADED = "NOT_GRADED" @property - def category(self) -> Literal["succeeded", "failed", "error"]: + def category(self) -> Literal["succeeded", "failed", "error", "ungraded"]: """Classify this status into a reporting category (the SSOT for failed/succeeded/error).""" return _STATUS_CATEGORIES[self] @@ -31,7 +38,7 @@ def icon(self) -> str: # catch-all default) so a newly-added status fails the assert below until it is # classified — rather than silently collapsing into "failed" (which would skew # reports AND the telemetry Category dimension). Mirrors the _STATUS_ICONS guard. -_STATUS_CATEGORIES: dict[FinalStatus, Literal["succeeded", "failed", "error"]] = { +_STATUS_CATEGORIES: dict[FinalStatus, Literal["succeeded", "failed", "error", "ungraded"]] = { FinalStatus.SUCCESS: "succeeded", FinalStatus.FAILURE: "failed", FinalStatus.ERROR: "error", @@ -43,6 +50,12 @@ def icon(self) -> str: FinalStatus.MAX_TURNS_EXHAUSTED: "failed", FinalStatus.TOKEN_BUDGET_EXCEEDED: "failed", FinalStatus.COST_BUDGET_EXCEEDED: "failed", + # A fourth category, not a fold into one of the three. Folding into + # "failed" would depress every pass rate; folding into "succeeded" would + # invent verdicts; folding into "error" would report a healthy run as + # broken. Reporting surfaces exclude it from BOTH the numerator and the + # denominator of a pass rate — an ungraded task was never measured. + FinalStatus.NOT_GRADED: "ungraded", } assert set(_STATUS_CATEGORIES) == set(FinalStatus), "Missing category for FinalStatus member" @@ -57,6 +70,7 @@ def icon(self) -> str: FinalStatus.MAX_TURNS_EXHAUSTED: "M", FinalStatus.TOKEN_BUDGET_EXCEEDED: "#", FinalStatus.COST_BUDGET_EXCEEDED: "$", + FinalStatus.NOT_GRADED: "?", } assert set(_STATUS_ICONS) == set(FinalStatus), "Missing icon for FinalStatus member" diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py index fe6ae6ff..4c28982b 100644 --- a/src/coder_eval/models/experiment.py +++ b/src/coder_eval/models/experiment.py @@ -222,6 +222,13 @@ class VariantAggregate(BaseModel): # noqa: CE009 -- persisted result model; rou tasks_succeeded: int tasks_failed: int tasks_error: int + # Fourth bucket of the task_count invariant (see RunSummary.tasks_not_graded). + # Defaulted so experiment.json written before `coder-eval execute` still loads. + tasks_not_graded: int = Field( + default=0, + ge=0, + description="Tasks executed without grading (`coder-eval execute`). Excluded from pass_rate entirely.", + ) average_score: float average_duration: float total_tokens: int | None = None @@ -244,16 +251,25 @@ class VariantAggregate(BaseModel): # noqa: CE009 -- persisted result model; rou @model_validator(mode="after") def _check_task_count_invariant(self) -> VariantAggregate: - if self.tasks_succeeded + self.tasks_failed + self.tasks_error != self.tasks_run: - total = f"{self.tasks_succeeded} + {self.tasks_failed} + {self.tasks_error}" + buckets = self.tasks_succeeded + self.tasks_failed + self.tasks_error + self.tasks_not_graded + if buckets != self.tasks_run: + total = f"{self.tasks_succeeded} + {self.tasks_failed} + {self.tasks_error} + {self.tasks_not_graded}" raise ValueError(f"Task count invariant violated: {total} != {self.tasks_run}") return self + @property + def tasks_graded(self) -> int: + """Tasks actually measured — ``pass_rate``'s denominator.""" + return self.tasks_run - self.tasks_not_graded + @computed_field # type: ignore[prop-decorator] @property def pass_rate(self) -> float | None: - """``tasks_succeeded / tasks_run`` as a 0-1 fraction. ``None`` on an empty variant.""" - return self.tasks_succeeded / self.tasks_run if self.tasks_run else None + """``tasks_succeeded / tasks_graded`` as a 0-1 fraction. ``None`` when nothing was graded. + + Mirrors ``RunSummary.pass_rate``: ungraded tasks leave both sides. + """ + return self.tasks_succeeded / self.tasks_graded if self.tasks_graded else None class TaskExperimentSummary(BaseModel): # noqa: CE009 -- persisted result model; round-trip leniency like models/results.py diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 8834710a..170b868b 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -1050,6 +1050,17 @@ class RunSummary(BaseModel): tasks_succeeded: int = Field(description="Number of tasks that succeeded") tasks_failed: int = Field(description="Number of tasks that failed") tasks_error: int = Field(description="Number of tasks that encountered errors") + # Part of the task_count invariant (a fourth bucket, not a sub-counter), but + # defaulted so run.json written before `coder-eval execute` existed — where + # no task can be ungraded — still deserialises. + tasks_not_graded: int = Field( + default=0, + ge=0, + description=( + "Number of tasks executed without grading (`coder-eval execute`). " + "Excluded from BOTH sides of pass_rate — an ungraded task was never measured." + ), + ) # Informational sub-counters: subsets of tasks_failed (NOT part of the # task_count invariant). Default 0 so old serialized RunSummary JSON @@ -1097,11 +1108,17 @@ class RunSummary(BaseModel): @model_validator(mode="after") def _check_task_count_invariant(self) -> RunSummary: - if self.tasks_succeeded + self.tasks_failed + self.tasks_error != self.tasks_run: - total = f"{self.tasks_succeeded} + {self.tasks_failed} + {self.tasks_error}" + buckets = self.tasks_succeeded + self.tasks_failed + self.tasks_error + self.tasks_not_graded + if buckets != self.tasks_run: + total = f"{self.tasks_succeeded} + {self.tasks_failed} + {self.tasks_error} + {self.tasks_not_graded}" raise ValueError(f"Task count invariant violated: {total} != {self.tasks_run}") return self + @property + def tasks_graded(self) -> int: + """Tasks that were actually measured — the denominator for every rate below.""" + return self.tasks_run - self.tasks_not_graded + # Derived run metrics: computed_fields over the stored counts and # ``task_results``, so they serialize into run.json while staying impossible to # set to something the rows disagree with. Consumers should read these rather @@ -1110,18 +1127,24 @@ def _check_task_count_invariant(self) -> RunSummary: @computed_field # type: ignore[prop-decorator] @property def pass_rate(self) -> float | None: - """``tasks_succeeded / tasks_run`` as a 0-1 fraction. ``None`` on an empty run.""" - return self.tasks_succeeded / self.tasks_run if self.tasks_run else None + """``tasks_succeeded / tasks_graded`` as a 0-1 fraction. ``None`` on an empty run. + + The denominator excludes ungraded tasks (``coder-eval execute``), which were + never measured — counting them as misses would report a clean execute run as + 0% pass. Identical to ``tasks_run`` for every graded run. + """ + return self.tasks_succeeded / self.tasks_graded if self.tasks_graded else None @computed_field # type: ignore[prop-decorator] @property def error_share(self) -> float | None: - """``tasks_error / tasks_run`` as a 0-1 fraction. ``None`` on an empty run. + """``tasks_error / tasks_graded`` as a 0-1 fraction. ``None`` on an empty run. Diagnostic only, never adjusts the rate: a drop at a high error share is an - infrastructure night, the same drop at a normal share is the model. + infrastructure night, the same drop at a normal share is the model. Shares + ``pass_rate``'s denominator so the two are directly comparable. """ - return self.tasks_error / self.tasks_run if self.tasks_run else None + return self.tasks_error / self.tasks_graded if self.tasks_graded else None @computed_field # type: ignore[prop-decorator] @property diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py index 8576cdfc..d79e6d96 100644 --- a/src/coder_eval/orchestration/batch.py +++ b/src/coder_eval/orchestration/batch.py @@ -164,6 +164,7 @@ async def run_single(rt: ResolvedTask) -> TaskResult: preservation_mode=preservation_mode, stream_callback=task_callback, verbose=config.verbose, + grade=config.grade, ).run() # The in-container _finalize_result can't emit task telemetry # (connection-string env vars aren't forwarded into the @@ -185,6 +186,7 @@ async def run_single(rt: ResolvedTask) -> TaskResult: source_yaml=rt.source_yaml, config_lineage=rt.config_lineage, replicate_index=rt.replicate_index, + grade=config.grade, ) result = await orchestrator.run() tr = TaskResult( @@ -665,6 +667,7 @@ def build_run_summary( tasks_succeeded=sum(1 for s in statuses if s.category == "succeeded"), tasks_failed=sum(1 for s in statuses if s.category == "failed"), tasks_error=sum(1 for s in statuses if s.category == "error"), + tasks_not_graded=sum(1 for s in statuses if s.category == "ungraded"), tasks_token_budget_exceeded=sum(1 for s in statuses if s == FinalStatus.TOKEN_BUDGET_EXCEEDED), tasks_cost_budget_exceeded=sum(1 for s in statuses if s == FinalStatus.COST_BUDGET_EXCEEDED), skipped_tasks=skipped_tasks or [], diff --git a/src/coder_eval/orchestration/config.py b/src/coder_eval/orchestration/config.py index 695b5594..80a3d812 100644 --- a/src/coder_eval/orchestration/config.py +++ b/src/coder_eval/orchestration/config.py @@ -94,6 +94,20 @@ class BatchRunConfig(BaseModel): description="CLI override for replicates per (task, variant). None = defer to experiment layers.", ) + # Grading switch: `coder-eval run` (True) vs `coder-eval execute` (False). + # It lives HERE and nowhere else on purpose — it is deliberately NOT part of + # the 5-layer task merge, so there is no `-D grade=...` path and no + # MergeField (CE014 does not apply to a scalar bool outside the merged + # roots). A task YAML must never be able to declare itself ungraded; only + # the invoking command decides. + grade: bool = Field( + default=True, + description=( + "Evaluate success criteria after execution. False = `coder-eval execute`: " + "run and capture the trajectory, score nothing, finalize as NOT_GRADED." + ), + ) + # Logging verbose: bool = Field(default=False, description="Enable verbose (DEBUG level) logging for Docker output") diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index 25d4ed58..3e8c1500 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -819,11 +819,22 @@ def _pick_worst_status(statuses: list[FinalStatus]) -> FinalStatus: Unknown categories fall back to priority -1 so they sort as worst-of-all (fail-closed: a new unrecognised status becomes the most urgent). + + "ungraded" sorts LEAST urgent (above "succeeded") — it carries no verdict, so + any replicate that does have one must win. It therefore survives only when + every replicate is ungraded, which is the only case reachable today anyway + (``grade`` is run-level, so replicates never mix). """ - priority = {"error": 0, "failed": 1, "succeeded": 2} + priority = {"error": 0, "failed": 1, "succeeded": 2, "ungraded": 3} return min(statuses, key=lambda s: priority.get(s.category, -1)) +def _mean_graded_score(vr_list: list[VariantResult]) -> float: + """Mean ``weighted_score`` over the graded rows; ``0.0`` when none were graded.""" + graded = [v.weighted_score for v in vr_list if v.final_status.category != "ungraded"] + return sum(graded) / len(graded) if graded else 0.0 + + def _mean_reference_similarity(reps: list[TaskResult]) -> float | None: """Return the mean reference_comparison score across replicates that have one.""" scores = [ @@ -938,9 +949,14 @@ def aggregate_results( tasks_succeeded=sum(1 for v in vr_list if v.final_status.category == "succeeded"), tasks_failed=sum(1 for v in vr_list if v.final_status.category == "failed"), tasks_error=sum(1 for v in vr_list if v.final_status.category == "error"), + tasks_not_graded=sum(1 for v in vr_list if v.final_status.category == "ungraded"), tasks_token_budget_exceeded=sum(1 for v in vr_list if v.final_status == FinalStatus.TOKEN_BUDGET_EXCEEDED), tasks_cost_budget_exceeded=sum(1 for v in vr_list if v.final_status == FinalStatus.COST_BUDGET_EXCEEDED), - average_score=sum(v.weighted_score for v in vr_list) / len(vr_list), + # Mean over GRADED rows only. An ungraded row has no score (it + # arrives here as 0.0 because VariantResult.weighted_score is a + # plain float), so including it would report a clean execute run as + # average_score 0.0 — a number indistinguishable from "scored zero". + average_score=_mean_graded_score(vr_list), average_duration=sum(v.duration_seconds / v.replicate_count for v in vr_list) / len(vr_list), total_tokens=total_tokens, replicate_count=vr_list[0].replicate_count if vr_list else 1, diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index a2697bc0..22092c17 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -363,6 +363,7 @@ def __init__( config_lineage: dict[str, ConfigLineageEntry] | None = None, replicate_index: int = 0, workspace_dir: Path | None = None, + grade: bool = True, ): """Initialize the orchestrator. @@ -385,6 +386,13 @@ def __init__( run_dir/artifacts/, and the workspace is copied out to run_dir/artifacts/ at cleanup. Resolved host-side by DockerRunner; None keeps standard behavior. Takes precedence over preservation_mode when set. + grade: Whether to evaluate success criteria after execution. False is + `coder-eval execute`: the agent runs and the full trajectory is + captured, but no criterion is checked, ``weighted_score`` stays + None, and the row finalizes as ``FinalStatus.NOT_GRADED``. It is + deliberately NOT a task-config field — a task YAML must never be + able to declare itself ungraded — so it arrives only from + ``BatchRunConfig.grade``, never from the 5-layer merge or -D. """ self.task = task self.run_dir = run_dir @@ -403,6 +411,7 @@ def __init__( self.source_yaml = source_yaml self.config_lineage = config_lineage or {} self.replicate_index = replicate_index + self.grade = grade # Derived paths self.report_path = self.run_dir / "task.json" @@ -583,11 +592,19 @@ def _kill_agent_subprocess_sync() -> None: elapsed_seconds=time.time() - start_time, ) - # Update final status + # Update final status. The NOT_GRADED arm sits between the + # execution facts and FAILURE deliberately: under grade=False no + # criterion ran, so `success` is always False and FAILURE would be + # a verdict we never actually reached — but MAX_TURNS_EXHAUSTED + # (like TIMEOUT / BUILD_FAILED / the budget stops on the except + # branches below) is a fact about the RUN, not about grading, and + # still applies. With grade=True the chain is unchanged. if success: self.result.final_status = FinalStatus.SUCCESS elif self.result.max_turns_exhausted: self.result.final_status = FinalStatus.MAX_TURNS_EXHAUSTED + elif not self.grade: + self.result.final_status = FinalStatus.NOT_GRADED else: self.result.final_status = FinalStatus.FAILURE @@ -683,6 +700,9 @@ def _kill_agent_subprocess_sync() -> None: # but BEFORE _finalize_result so task.json includes the field. # Allowlist non-success terminal statuses; SUCCESS and # MAX_TURNS_EXHAUSTED skip the tail to keep task.json compact. + # NOT_GRADED is deliberately absent: like SUCCESS and + # MAX_TURNS_EXHAUSTED it is not a diagnosis of something going + # wrong, so it keeps task.json compact. if self.result.final_status in { FinalStatus.ERROR, FinalStatus.TIMEOUT, @@ -785,6 +805,11 @@ async def _evaluate_post_failure_criteria(self) -> None: """ if self.result is None: return + if not self.grade: + # Grading site 4 of 4. Under `execute` no criterion is checked on any + # path, diagnostics included — recording a not_evaluated vector here + # would imply criteria we were supposed to run and couldn't. + return if self.success_checker is None or self.sandbox is None: self._record_post_failure_not_evaluated("the sandbox or success checker was unavailable") return @@ -879,7 +904,14 @@ def _finalize_result(self, start_time: float) -> None: # path) run inside run()'s try, whose broad `except Exception` already converts # a raise into a populated ERROR result, so they intentionally stay unwrapped. try: - self.result.calculate_weighted_score(self.task.success_criteria) + if self.grade: + self.result.calculate_weighted_score(self.task.success_criteria) + else: + # Explicit None, NOT the 0.0 calculate_weighted_score writes for an + # empty results list — that value is indistinguishable from a task + # that was graded and scored zero, and every downstream `score or + # 0.0` would launder it into a real-looking failure. + self.result.weighted_score = None except ValueError as e: logger.error("Weighted-score computation failed; marking row ERROR: %s", e, exc_info=True) self.result.weighted_score = None @@ -1284,7 +1316,18 @@ async def _setup(self) -> None: # armed evaluate-only re-grade builds an inert (never-fed) watcher — # harmless, and keeps a single creation point. if early_stop_active(self.task): - self._early_stop_watcher = EarlyStopWatcher.for_task(self.task) + if self.grade: + self._early_stop_watcher = EarlyStopWatcher.for_task(self.task) + else: + # Early stop cuts the run once coder-eval's own criteria decide the + # outcome. Under `execute` there is no outcome to decide and the + # trajectory is the deliverable (an external harness grades it), so + # an armed criterion must not truncate it. Same effect as the + # run_limits.stop_early kill switch, decided one layer up. + logger.info( + "Grading disabled (execute mode): early-stop is armed but stays disabled; " + + "the full trajectory is the deliverable." + ) # Stage the reference BEFORE either branch returns: judge criteria with # include_reference=true (and any $REFERENCE_DIR/... file entry) expect it @@ -1888,6 +1931,14 @@ async def _evaluation_loop(self) -> bool: assert self.task.agent is not None if self.agent is None: + # Grading site 1 of 4. Evaluate-only with grading off would neither + # run an agent nor check anything — a no-op that still writes a + # task.json. Refuse instead of producing an empty row. + if not self.grade: + raise ValueError( + "grade=False is meaningless on the evaluate-only path (no agent attached): " + + "the run would neither execute nor grade." + ) # No agent attached: evaluate-only re-grade of a completed sandbox. # (No-op tasks have a NoOpAgent here, so they take the normal path # below.) Check the criteria directly against the sandbox. @@ -1955,6 +2006,15 @@ async def _evaluation_loop(self) -> bool: logger.debug(f"Agent response received ({len(turn_record.agent_output)} chars)") + # Grading site 2 of 4. `execute` stops here: the trajectory is captured + # and persisted exactly as on a graded run, but nothing is scored. + # Returning False keeps FinalStatus off SUCCESS; run()'s status chain + # turns it into NOT_GRADED. The reference-integrity check is skipped too + # — it exists to protect a grade that is not happening. + if not self.grade: + logger.info("Grading disabled (execute mode): skipping success criteria.") + return False + # Check success criteria (reference_dir feeds reference_comparison + judges) logger.debug("Checking success criteria") await self._verify_reference_integrity() @@ -2073,6 +2133,15 @@ async def _run_dialog_criteria_check( """ assert self.result is not None assert self.success_checker is not None + # Grading site 3 of 4. Unreachable today — `execute` rejects simulation + # tasks at the CLI, because the dialog's turn-continuation logic reads + # criteria results to decide whether to keep talking, so an ungraded + # dialog would silently change its own stopping behavior. Kept as a + # correct, defensive no-op so the gate holds if that restriction lifts. + if not self.grade: + self.result.success_criteria_results = [] + self.result.weighted_score = None + return [] await self._verify_reference_integrity() criteria_results = await self.success_checker.check_all_async( self.task.success_criteria, diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py index 88345636..ea6d5897 100644 --- a/src/coder_eval/reports.py +++ b/src/coder_eval/reports.py @@ -237,8 +237,16 @@ def early_stop_gate_note(reason: str) -> str: def _pass_rate_lines(summary: RunSummary) -> list[str]: - """The pass rate over every dispatched task, plus the error share when non-zero.""" - lines = [f"- **Pass Rate**: {_fmt_rate(summary.pass_rate)} ({summary.tasks_succeeded}/{summary.tasks_run})"] + """The pass rate over every GRADED task, plus the error share when non-zero. + + An ungraded run (``coder-eval execute``) has no pass rate at all, so it says + so rather than rendering ``0.0% (0/N)`` — which reads as a total failure. + """ + # Only an ungraded run gets the explanatory line. An ordinary EMPTY run keeps + # its original "n/a (0/0)" rendering — the two are different facts. + if summary.tasks_not_graded and not summary.tasks_graded: + return [f"- **Pass Rate**: n/a — {summary.tasks_not_graded} task(s) executed without grading"] + lines = [f"- **Pass Rate**: {_fmt_rate(summary.pass_rate)} ({summary.tasks_succeeded}/{summary.tasks_graded})"] if summary.tasks_error: lines.append( f"- **Error Share**: {_fmt_rate(summary.error_share)} of tasks never produced a " @@ -389,6 +397,7 @@ def _summary_section_lines(summary: RunSummary) -> list[str]: f"- **Succeeded**: {summary.tasks_succeeded}", failed_line, f"- **Errors**: {summary.tasks_error}", + *([f"- **Not Graded**: {summary.tasks_not_graded}"] if summary.tasks_not_graded else []), *_pass_rate_lines(summary), ] diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index e141174b..e3cdd95f 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -626,7 +626,10 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir: f"- **Succeeded**: {agg.tasks_succeeded}", failed_line, f"- **Errors**: {agg.tasks_error}", - f"- **Pass Rate**: {pass_rate_str} ({agg.tasks_succeeded}/{agg.tasks_run})", + *([f"- **Not Graded**: {agg.tasks_not_graded}"] if agg.tasks_not_graded else []), + # Denominator is the GRADED count, matching VariantAggregate.pass_rate — + # an ungraded task was never measured and belongs on neither side. + f"- **Pass Rate**: {pass_rate_str} ({agg.tasks_succeeded}/{agg.tasks_graded})", f"- **Average Score**: {agg.average_score:.3f}", f"- **Average Duration**: {agg.average_duration:.1f}s", f"- **Total Tokens**: {tokens_str}", diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py index def6591f..7f5d4468 100644 --- a/src/coder_eval/reports_html.py +++ b/src/coder_eval/reports_html.py @@ -283,7 +283,10 @@ def _status_badge(status: Any) -> str: status_str = getattr(status, "value", None) or str(status) try: fs = status if isinstance(status, FinalStatus) else FinalStatus(str(status)) - cls = {"succeeded": "success", "failed": "failure", "error": "error"}[fs.category] + # "ungraded" -> neutral: the row carries no verdict, so it must render as + # neither green nor red. Same class an unrecognised status falls back to, + # reached deliberately here rather than by accident. + cls = {"succeeded": "success", "failed": "failure", "error": "error", "ungraded": "neutral"}[fs.category] except (ValueError, KeyError): cls = "neutral" # unknown / non-FinalStatus input return f'{_esc(status_str)}' diff --git a/src/coder_eval/reports_junit.py b/src/coder_eval/reports_junit.py index 133f9187..acc609a7 100644 --- a/src/coder_eval/reports_junit.py +++ b/src/coder_eval/reports_junit.py @@ -54,7 +54,7 @@ def _xml_safe(text: str) -> str: return _ILLEGAL_XML.sub("", text) -def _category_of(status: str) -> Literal["succeeded", "failed", "error"]: +def _category_of(status: str) -> Literal["succeeded", "failed", "error", "ungraded"]: """Map a serialized status string to a reporting category via the SSOT. Goes through ``FinalStatus(value).category`` (an explicit allowlist, CE018); @@ -303,6 +303,14 @@ def _task_case(row: dict[str, Any], run_dir: Path) -> ET.Element: if category == "succeeded": return case + if category == "ungraded": + # `coder-eval execute`: the task ran but was deliberately not scored. + # is JUnit's only "no verdict" element — reporting it as a + # would turn a healthy ungraded run red in CI, and reporting + # it as a pass would invent a verdict. _set_counts already counts these. + ET.SubElement(case, "skipped", {"message": "not graded (coder-eval execute)"}) + return case + message = status if status in _KNOWN_STATUSES else f"unknown status: {status}" tag = "failure" if category == "failed" else "error" child = ET.SubElement(case, tag, {"message": _xml_safe(message)}) diff --git a/tests/lint/rules/ce018_no_final_status_name_denylist.py b/tests/lint/rules/ce018_no_final_status_name_denylist.py index 2e9c12a1..1a3b4f96 100644 --- a/tests/lint/rules/ce018_no_final_status_name_denylist.py +++ b/tests/lint/rules/ce018_no_final_status_name_denylist.py @@ -35,6 +35,7 @@ "MAX_TURNS_EXHAUSTED", "TOKEN_BUDGET_EXCEEDED", "COST_BUDGET_EXCEEDED", + "NOT_GRADED", } ) diff --git a/tests/test_execute_command.py b/tests/test_execute_command.py new file mode 100644 index 00000000..5f0f02da --- /dev/null +++ b/tests/test_execute_command.py @@ -0,0 +1,293 @@ +"""`coder-eval execute` — run without grading. + +Three layers, deliberately: + +* **End-to-end** against the agentless task (`agent: {type: none}`), which needs + no API key and is fully deterministic. This is the only layer that proves the + whole chain — CLI → batch → Orchestrator → task.json → run.json — actually + withholds the verdict while still executing. +* **Contrast** — the same task under `run` must still produce SUCCESS with a real + score. Without it, a totally broken `execute` (or a broken fixture) would pass + the assertions above by accident. +* **Wiring** — the two commands share one body, so a signature or `grade` drift + is caught mechanically rather than by a human noticing. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any +from unittest.mock import patch + +import pytest +from typer.testing import CliRunner + +from coder_eval.cli import app +from coder_eval.models import FinalStatus, RunSummary + + +runner = CliRunner() + +# The agentless smoke task: no agent, no model call, and a pre_run that writes a +# file its criteria read back. Executing it must still write that file (proving +# the run really happened) while scoring nothing. +AGENTLESS_TASK = Path("tasks/agentless_smoke_test.yaml") + + +def _invoke(command: str, run_dir: Path) -> Any: + return runner.invoke( + app, + [command, str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--preservation-mode", "MOVE_ON_WRITE"], + ) + + +def _task_json(run_dir: Path) -> dict[str, Any]: + matches = sorted(run_dir.glob("**/task.json")) + assert len(matches) == 1, f"expected exactly one task.json under {run_dir}, got {matches}" + return json.loads(matches[0].read_text(encoding="utf-8")) + + +# -------------------------------------------------------------------------- +# The status itself +# -------------------------------------------------------------------------- + + +def test_not_graded_is_its_own_category() -> None: + """NOT_GRADED must not fold into succeeded/failed/error — each would lie.""" + assert FinalStatus.NOT_GRADED.category == "ungraded" + assert FinalStatus.NOT_GRADED.icon == "?" + + +def test_ungraded_leaves_both_sides_of_the_pass_rate() -> None: + """An all-ungraded run has NO pass rate — not a 0% one.""" + summary = RunSummary( + run_id="r", + start_time="2026-01-01T00:00:00", # type: ignore[arg-type] + end_time="2026-01-01T00:01:00", # type: ignore[arg-type] + total_duration_seconds=60.0, + tasks_run=2, + tasks_succeeded=0, + tasks_failed=0, + tasks_error=0, + tasks_not_graded=2, + task_results=[], + framework_version="test", + ) + assert summary.tasks_graded == 0 + assert summary.pass_rate is None + assert summary.error_share is None + + +def test_ungraded_does_not_dilute_a_partially_graded_run() -> None: + """One pass out of one graded task is 100%, even alongside three ungraded ones.""" + summary = RunSummary( + run_id="r", + start_time="2026-01-01T00:00:00", # type: ignore[arg-type] + end_time="2026-01-01T00:01:00", # type: ignore[arg-type] + total_duration_seconds=60.0, + tasks_run=4, + tasks_succeeded=1, + tasks_failed=0, + tasks_error=0, + tasks_not_graded=3, + task_results=[], + framework_version="test", + ) + assert summary.pass_rate == 1.0 + + +def test_task_count_invariant_counts_the_ungraded_bucket() -> None: + """The fourth bucket is part of the invariant, not a free-floating sub-counter.""" + with pytest.raises(ValueError, match="Task count invariant violated"): + RunSummary( + run_id="r", + start_time="2026-01-01T00:00:00", # type: ignore[arg-type] + end_time="2026-01-01T00:01:00", # type: ignore[arg-type] + total_duration_seconds=1.0, + tasks_run=2, + tasks_succeeded=0, + tasks_failed=0, + tasks_error=0, + tasks_not_graded=1, # 0+0+0+1 != 2 + task_results=[], + framework_version="test", + ) + + +# -------------------------------------------------------------------------- +# End to end +# -------------------------------------------------------------------------- + + +@pytest.mark.skipif(not AGENTLESS_TASK.is_file(), reason="needs a source checkout (tasks/ is not in the wheel)") +def test_execute_runs_the_task_but_grades_nothing(tmp_path: Path) -> None: + run_dir = tmp_path / "run" + result = _invoke("execute", run_dir) + + assert result.exit_code == 0, result.output + + row = _task_json(run_dir) + # The verdict is withheld ... + assert row["final_status"] == FinalStatus.NOT_GRADED.value + assert row["weighted_score"] is None, "must be None, never 0.0 — 0.0 reads as 'graded and scored zero'" + assert row["success_criteria_results"] == [] + # ... but the run itself demonstrably happened: pre_run wrote its file into + # the preserved sandbox. Without this the test would also pass if `execute` + # had simply skipped the task. + proof = sorted(run_dir.glob("**/artifacts/**/proof.txt")) + assert proof, f"pre_run did not run — no proof.txt under {run_dir}" + assert "coder-eval-ran-without-a-coder" in proof[0].read_text(encoding="utf-8") + + +@pytest.mark.skipif(not AGENTLESS_TASK.is_file(), reason="needs a source checkout (tasks/ is not in the wheel)") +def test_execute_run_json_reports_ungraded_not_failed(tmp_path: Path) -> None: + run_dir = tmp_path / "run" + assert _invoke("execute", run_dir).exit_code == 0 + + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert summary["tasks_run"] == 1 + assert summary["tasks_not_graded"] == 1 + # The whole point: an ungraded task is NOT a failure and NOT an error. + assert summary["tasks_failed"] == 0 + assert summary["tasks_error"] == 0 + assert summary["tasks_succeeded"] == 0 + assert summary["pass_rate"] is None + + +@pytest.mark.skipif(not AGENTLESS_TASK.is_file(), reason="needs a source checkout (tasks/ is not in the wheel)") +def test_run_still_grades_the_same_task(tmp_path: Path) -> None: + """The negative control: `run` must still score this task, or the assertions + above prove nothing about grading being *deliberately* skipped.""" + run_dir = tmp_path / "run" + result = _invoke("run", run_dir) + + assert result.exit_code == 0, result.output + row = _task_json(run_dir) + assert row["final_status"] == FinalStatus.SUCCESS.value + assert row["weighted_score"] == 1.0 + assert len(row["success_criteria_results"]) == 2 + + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert summary["tasks_not_graded"] == 0 + assert summary["tasks_succeeded"] == 1 + assert summary["pass_rate"] == 1.0 + + +# -------------------------------------------------------------------------- +# Wiring: one shared body, two commands +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("command", "module", "expected_grade"), + [("run", "run_command", True), ("execute", "execute_command", False)], +) +def test_both_commands_call_the_shared_pipeline(command: str, module: str, expected_grade: bool) -> None: + """`run` and `execute` differ ONLY in `grade` — no third code path. + + Patched per module because each command imported ``run_pipeline`` into its own + namespace; patching the defining module would silently miss ``execute``. + """ + with patch(f"coder_eval.cli.{module}.run_pipeline") as pipeline: + result = runner.invoke(app, [command, "a.yaml"]) + assert result.exit_code == 0, result.output + pipeline.assert_called_once() + assert pipeline.call_args.kwargs["grade"] is expected_grade + + +def _option_names(command: str) -> set[str]: + import typer.main + + click_app = typer.main.get_command(app) + cmd = click_app.commands[command] # type: ignore[attr-defined] + return {opt for param in cmd.params for opt in getattr(param, "opts", [])} + + +# `execute` restates `run`'s Typer signature because Typer builds its parser from +# the signature and there is no way to share one. That duplication is the drift +# risk this test exists to close: a flag added to `run` must be added here too, +# or consciously listed below as a deliberate omission. +_DELIBERATELY_ABSENT_FROM_EXECUTE = { + "--resume", # partition_for_resume would treat a NOT_GRADED row as finalized + "--junit-xml", # a report of verdicts, and there are none +} + + +def test_execute_exposes_run_flags_minus_the_two_refused_ones() -> None: + run_opts = _option_names("run") + execute_opts = _option_names("execute") + + missing = run_opts - execute_opts - _DELIBERATELY_ABSENT_FROM_EXECUTE + assert not missing, ( + f"`run` has flag(s) {sorted(missing)} that `execute` lacks. Add them to " + "execute_command's signature, or list them in _DELIBERATELY_ABSENT_FROM_EXECUTE " + "with the reason they are refused." + ) + assert not execute_opts - run_opts, "`execute` must not grow flags of its own" + # The omissions must be real, not stale entries masking a genuine gap. + assert not _DELIBERATELY_ABSENT_FROM_EXECUTE & execute_opts + + +# -------------------------------------------------------------------------- +# The docker boundary +# -------------------------------------------------------------------------- + + +async def _staged_context(tmp_path: Path, *, grade: bool) -> dict[str, Any]: + """Stage a docker task's inputs and read back the context.json the container sees.""" + from coder_eval.isolation.docker_runner import DockerRunner + from coder_eval.models import ResolvedTask, TaskDefinition + + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent={"type": "claude-code"}, + sandbox={"driver": "docker"}, + success_criteria=[{"type": "file_exists", "path": "x.txt", "description": "x"}], + ) + rt = ResolvedTask( + task=task, + task_file=tmp_path / "t.yaml", + run_dir=tmp_path / "run", + variant_id="default", + original_task_id="t", + ) + staged = tmp_path / "input" + staged.mkdir() + await DockerRunner(rt, grade=grade)._stage_inputs(staged) + return json.loads((staged / "context.json").read_text(encoding="utf-8")) + + +@pytest.mark.parametrize("grade", [True, False]) +async def test_docker_forwards_grade_to_the_container(tmp_path: Path, grade: bool) -> None: + """`grade` is a run-level CLI decision, so it is NOT recoverable from the staged + task.yaml on the container side — it has to cross the boundary in context.json. + Without this, `execute --driver docker` would silently grade after all.""" + assert (await _staged_context(tmp_path, grade=grade))["grade"] is grade + + +def test_container_defaults_to_grading_when_the_host_sends_no_key() -> None: + """A host predating `execute` writes no `grade` key; the container must keep + its original (grading) behavior rather than silently withholding verdicts.""" + # The parse is inline in a Typer command that cannot run outside a container, + # so this reads its source. Resolved off the function object because + # `coder_eval.cli` rebinds the submodule's name to the function it exports. + import inspect + + from coder_eval.cli.run_task_internal_command import run_task_internal_command + + source = inspect.getsource(inspect.getmodule(run_task_internal_command)) # type: ignore[arg-type] + assert 'context.get("grade", True)' in source, "the in-container default must be True (grade)" + + +def test_execute_help_explains_the_refused_flags() -> None: + """The two omissions are documented in the help, not silently absent — a user + who reaches for `--resume` needs to learn why it is refused, not just that it + is unrecognised. (Presence as a real *flag* is covered by the option-set test + above; here we only require the help text to mention them.)""" + result = runner.invoke(app, ["execute", "--help"]) + assert result.exit_code == 0 + for flag in _DELIBERATELY_ABSENT_FROM_EXECUTE: + assert flag in result.output, f"execute's help should explain why {flag} is unavailable" diff --git a/tests/test_reports_html.py b/tests/test_reports_html.py index 77e5886a..17c77255 100644 --- a/tests/test_reports_html.py +++ b/tests/test_reports_html.py @@ -37,16 +37,21 @@ ) -_CATEGORY_TO_CLASS = {"succeeded": "success", "failed": "failure", "error": "error"} +# "ungraded" is the one category whose badge is legitimately neutral: the row +# carries no verdict, so it must render as neither green nor red. Listing it +# explicitly (rather than dropping the negative assertion below) keeps the guard +# that no OTHER member falls through to the neutral fallback. +_CATEGORY_TO_CLASS = {"succeeded": "success", "failed": "failure", "error": "error", "ungraded": "neutral"} @pytest.mark.parametrize("status", list(FinalStatus)) def test_status_badge_dispatches_on_category(status: FinalStatus): - """Every FinalStatus member renders a non-neutral, category-correct badge.""" + """Every FinalStatus member renders a category-correct badge, neutral only when intended.""" badge = _status_badge(status) expected_cls = _CATEGORY_TO_CLASS[status.category] assert f'class="badge {expected_cls}"' in badge - assert "neutral" not in badge + if expected_cls != "neutral": + assert "neutral" not in badge # The human-readable label is the status value. assert status.value in badge @@ -1229,16 +1234,20 @@ def test_generation_metrics_breaks_down_crashed_partials(): FinalStatus.MAX_TURNS_EXHAUSTED: "failure", FinalStatus.TOKEN_BUDGET_EXCEEDED: "failure", FinalStatus.COST_BUDGET_EXCEEDED: "failure", + # Neutral on purpose — an ungraded row has no verdict to colour. See + # _CATEGORY_TO_CLASS above. + FinalStatus.NOT_GRADED: "neutral", } @pytest.mark.parametrize("status", list(FinalStatus)) def test_status_badge_maps_every_member_to_its_category(status: FinalStatus): - """Every FinalStatus member gets a non-neutral badge matching its category.""" + """Every FinalStatus member gets a badge matching its category, neutral only when intended.""" badge = _status_badge(status) expected_cls = _EXPECTED_BADGE_CLASS[status] assert f'class="badge {expected_cls}"' in badge - assert "neutral" not in badge + if expected_cls != "neutral": + assert "neutral" not in badge assert status.value in badge From 39db36108c5f421aea0ec02e2ca6b559605a3843 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Thu, 3 Sep 2026 13:42:41 -0700 Subject: [PATCH 02/11] =?UTF-8?q?feat(cli):=20grade=20an=20executed=20run?= =?UTF-8?q?=20afterwards=20=E2=80=94=20`evaluate=20`=20+=20`San?= =?UTF-8?q?dbox.adopt`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `coder-eval execute` withholds the verdict; this closes the loop by letting `coder-eval evaluate` supply it later, and fixes a pre-existing bug that made the copy-based grading path score real files as missing. ## `evaluate` takes two shapes Told apart by a pure resolver (`cli/evaluate_target.py`) on one probe: a target holding `task.json` is a run directory. coder-eval evaluate tasks/hello.yaml ./my_solution # unchanged coder-eval evaluate ./r/default/hello/00 # re-grade a finished run coder-eval execute tasks/hello.yaml --run-dir ./r coder-eval evaluate ./r/default/hello/00 coder-eval aggregate ./r # run.json now reports the verdict Passing a task file OVER a run directory re-grades it with different criteria, reusing the trajectory and workspace of a run you already paid for. ## Re-grading must describe the run that happened Run-dir mode rebuilds the task from the run's own `task_config.resolved`, NOT by re-reading the YAML. `resolved` is post-merge, so variant overrides, -D flags and dataset row expansion are already baked in; re-loading the source would silently grade a different task. Falling back to `source_file` happens only when `resolved` no longer validates, and says so loudly. `Orchestrator(prior_result=...)` seeds the fresh result via `_seed_from_prior_result`, which carries: - the trajectory — every derived figure (tokens, cost, command_stats, model_used, assistant turns) recomputes from `iterations`, so seeding it reproduces them exactly; - `iteration_count`, which evaluate-only used to flatten to 1; - `early_stop` — LOAD-BEARING. Gate selection is FIRED-ONLY: when it is set the checker gates on the weighted ARMED subset instead of strict-AND. Dropping it would re-grade a truncated trajectory under the full-run gate and flip the verdict; - execution facts (max_turns_exhausted, error_message/details, sdk_options). Grader-host `environment_info` is preserved under a `graded_by` sub-dict rather than overwriting the run's — showing the grader's tool versions as the run's is worse than showing neither. Two further parity fixes, both closing gaps the code already knew about: - `command_base_path` is now persisted by `_sync_sandbox_command_path_with_ agent` and restored in the evaluate-only branch. That method's docstring named "evaluate-only mode" as a known PATH gap; without it a detached grade resolves `run_command` binaries against ambient PATH and can disagree with the run it claims to grade. - `_join_litellm_actual_cost` skips when `prior_result` is set. It keys on a per-Orchestrator nonce the prior turns were never tagged with, so it would match nothing and overwrite already-correct per-turn costs. A re-grade refuses outright on a `reference_digest` mismatch: grading then would score the agent's old work against a new answer key. The verdict is written back into the run's `task.json`, keeping the pre-grade record as `task.execute.json`. That in-place write is what makes plain `coder-eval aggregate ` rebuild a graded run.json with zero new code. ## `Sandbox.adopt` — and the bug it fixes `adopt(workspace)` reuses `setup`'s adoption half but skips every MATERIALIZING step (`_setup_template`, `_generate_cli_recorders`, venv/package installs, the destructive $HOME remediation), running only non-mutating derivation: mock-dir +x, venv *discovery*, plugin-tools pin. `_cleanup_on_exit` stays False, so an adopted tree is never moved or deleted. In-place is MORE CORRECT, not merely faster. `_setup_template` filters its copy through `_should_ignore_template_file`, whose default list drops node_modules, dist, build, .venv and .git. So `evaluate` today scores a file that is plainly there as missing: copy path: Score 0.00 "File 'node_modules/x/a.js' does not exist" in place: Score 1.00 "File 'node_modules/x/a.js' exists" That is a pre-existing defect independent of `execute`. Defaults: in-place for a run directory (it is the run's own output), copy for a bare work directory (criteria can mutate it and it is the user's tree); `--in-place` / `--copy` override. `adopt` hard-errors on `driver: docker` — a container workspace is unreachable from the host, so adopting one would grade whatever happens to sit at that host path. ## Also The Typer command is now a thin wrapper over `run_evaluation(...)`, which has real Python defaults — the same split `run`/`execute` use. Calling a Typer command function in-process hands unspecified options an `OptionInfo` sentinel, which silently made `in_place=None` truthy; the existing test_evaluate_command.py calls were the ones that surfaced it. ## Verification `make verify` green (4602 passed, 92.06%). The headline test asserts `execute` + `evaluate` reaches the same status, score and per-criterion results as a single `run` — compared against a real `run` rather than hardcoded values, so a change breaking both paths still fails. Plus: aggregate rebuilds a graded run.json unaided; the trajectory survives the re-grade; the adopted workspace is not moved or deleted; task.execute.json preserves the ungraded record; the original two-argument form still works; adopt writes nothing, deletes nothing, and exposes the filtered directories; and the target resolver is table-tested over every (one arg / two args) x (run dir / plain dir / file / missing) combination. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 3 + docs/USER_GUIDE.md | 53 +++- src/coder_eval/cli/evaluate_command.py | 367 ++++++++++++++++++++++--- src/coder_eval/cli/evaluate_target.py | 99 +++++++ src/coder_eval/orchestrator.py | 84 +++++- src/coder_eval/sandbox.py | 77 ++++++ tests/test_evaluate_command.py | 32 +-- tests/test_evaluate_target.py | 102 +++++++ tests/test_execute_evaluate_loop.py | 147 ++++++++++ tests/test_litellm_cost.py | 6 + tests/test_sandbox_adopt.py | 108 ++++++++ 11 files changed, 1007 insertions(+), 71 deletions(-) create mode 100644 src/coder_eval/cli/evaluate_target.py create mode 100644 tests/test_evaluate_target.py create mode 100644 tests/test_execute_evaluate_loop.py create mode 100644 tests/test_sandbox_adopt.py diff --git a/CLAUDE.md b/CLAUDE.md index 896c2e4b..20f8f5e0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -92,6 +92,8 @@ coder_eval/ │ ├── run_command.py # `coder-eval run` + `run_pipeline` (the body BOTH run and execute share) │ ├── execute_command.py # `coder-eval execute` — Typer signature only; delegates to run_pipeline(grade=False) │ ├── plan_command.py # `coder-eval plan` +│ ├── evaluate_command.py # `coder-eval evaluate` (grade a dir, or re-grade a run dir) + `run_evaluation` +│ ├── evaluate_target.py # PURE shape detection for evaluate's positionals (run dir ⟺ holds task.json) │ ├── report_command.py # `coder-eval report` │ ├── run_helpers.py # CLI helper functions │ ├── console.py # Rich console instance @@ -148,6 +150,7 @@ action.yml # Published composite GitHub Action (coder-ev - **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex and Antigravity (both run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced), and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. Full table + rationale: docs/agents/HARNESS_PARITY.md. - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly. - **Execute vs. run (the grading switch)**: `coder-eval execute` is `coder-eval run` with grading removed — the agent runs and the full trajectory is captured, but no criterion is checked, `weighted_score` is `None` (never `0.0`, which would be indistinguishable from "graded and scored zero"), and the row finalizes as **`FinalStatus.NOT_GRADED`**, whose `category` is a **fourth** bucket, `"ungraded"`. Ungraded rows leave BOTH sides of every rate: `RunSummary.pass_rate` / `error_share` and `VariantAggregate.pass_rate` divide by `tasks_graded` (`tasks_run - tasks_not_graded`), and `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter. **Only SUCCESS/FAILURE collapse into it** — `ERROR`, `TIMEOUT`, `BUILD_FAILED`, `MAX_TURNS_EXHAUSTED` and the budget stops are facts about the *run*, not about grading, and still apply (so `execute` still exits non-zero on a crash). The switch is `BatchRunConfig.grade` → `Orchestrator(grade=...)` → the **four** grading call sites (single-shot, evaluate-only, the simulation dialog check, and post-failure diagnostics); it crosses the docker boundary in `context.json` (defaulting to `True` in-container, so a host predating `execute` keeps grading). It is **deliberately not a task-config field** — no 5-layer merge, no `-D` path — because a task YAML must never declare itself ungraded; only the invoking command decides. `run` and `execute` share one body (`run_command.run_pipeline`) and differ solely in that flag, so there is no third code path. Three things are refused rather than degraded: `--junit-xml` (a report of verdicts, and there are none — though `reports_junit` still emits `` for an ungraded row it encounters), `--resume` (`partition_for_resume` treats "has any final status" as finalized, so a `NOT_GRADED` row would be *skipped* by a later `run --resume` instead of graded), and simulation tasks (their turn-continuation logic reads criteria results, so an ungraded dialog would silently change its own stopping behavior). `stop_early:` blocks are inert under `execute` for the same reason the kill switch exists: the full trajectory is the deliverable. Motivating consumer: an external harness (Harbor / Terminal-Bench 2.0) that builds its own container, calls coder-eval as the agent, and grades with its own tests. +- **Detached grading (`evaluate` over a run dir) + `Sandbox.adopt`**: `coder-eval evaluate` takes two shapes, told apart by a **pure** resolver (`cli/evaluate_target.py`) on one probe — a target holding `task.json` is a run directory. Run-dir mode rebuilds the task from the run's own `task_config.resolved`, **not** by re-loading the YAML: `resolved` is post-merge, so variant overrides / `-D` / dataset expansion are already baked in and re-loading the source would silently grade a *different* task (fallback to `source_file` only when `resolved` no longer validates, and loudly). It seeds the fresh result from the prior one via `Orchestrator(prior_result=...)` → `_seed_from_prior_result`, which carries the trajectory (every derived figure — tokens, cost, `command_stats`, `model_used` — recomputes from `iterations`), `iteration_count`, execution facts, and **`early_stop` — load-bearing, because gate selection is FIRED-ONLY**: dropping it re-grades a truncated trajectory under the full-run strict-AND gate and can flip the verdict. Grader-host `environment_info` is preserved under a `graded_by` sub-dict rather than overwriting the run's. Two other parity fixes: `command_base_path` is now persisted into `environment_info` by `_sync_sandbox_command_path_with_agent` and restored in the evaluate-only branch (closing the PATH gap that method's docstring already named), and `_join_litellm_actual_cost` **skips** when `prior_result` is set (its join keys on a per-Orchestrator nonce the prior turns never carried, so it would clobber already-correct costs). The verdict is written back into the run's `task.json`, with the pre-grade record kept as `task.execute.json` — that in-place write is what makes plain `coder-eval aggregate ` rebuild a graded `run.json` with **zero** new code. **`Sandbox.adopt(workspace)`** is the grade-in-place primitive: it reuses `setup`'s adoption half but skips every *materializing* step (`_setup_template`, `_generate_cli_recorders`, venv/package installs, the destructive `$HOME` remediation), running only non-mutating derivation (mock-dir `+x`, venv *discovery*, plugin-tools pin); `_cleanup_on_exit` stays False so an adopted tree is never moved or deleted. In-place is **more correct**, not merely faster: `_setup_template` filters the copy through `_should_ignore_template_file`, which drops `node_modules` / `dist` / `build` / `.venv` / `.git`, so on the copy path a criterion like `test -f dist/bundle.js` fails as a *copying artifact* rather than as a verdict (verified: 0.00 "does not exist" on copy vs 1.00 in place). Defaults: in-place for a run dir, copy for a bare work dir (criteria can mutate it and it is the user's own tree); `--in-place`/`--copy` override. `adopt` hard-errors on `driver: docker` (a container workspace is unreachable from the host) and a re-grade refuses on a `reference_digest` mismatch. NOTE the Typer command is a thin wrapper over `run_evaluation(...)`, which has real Python defaults — calling a Typer command function in-process hands unspecified options an `OptionInfo` sentinel, which silently made `in_place=None` truthy. - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `stop_early.decide_within`, deliberately lives on `LiveSuccessCriterion` instead (see below) — the watcher must attribute a decision-step timeout to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express. - **Early stop on criterion (opt-in, per-criterion arming)**: a `stop_early:` block (`StopEarlyPolicy`) on a criterion ends a single-shot run early once the run's **armed** criteria decide the outcome, so a raised `max_turns` isn't wasted on the smoke flavor. The block's PRESENCE is the arming and alone activates the watcher — there is **no run-level master switch**: `run_limits.stop_early: false` is the run-level KILL SWITCH that force-disarms every block (the one-line experiment-variant/`-D` override for an authoritative full run), and `run_limits.stop_early: true` (the removed master arm) is a hard `EarlyStopConfigError` at resolution. The block exists on `LiveSuccessCriterion` only (currently `skill_triggered`, `command_executed` — so arming an unobservable criterion is unrepresentable, a pydantic extra-forbid error). Arming carries one implicit trigger (a native live-fail may fail-stop the run); its keys refine it: `on_pass: stop` (pass-stop the moment the criterion live-passes; default `continue` just latches) and `decide_within: N` (still undecided after N tool-call steps latches an **effective fail**, fed through the same fail-stop rule, reported as `decision_budget_exceeded` — an ordinary weighted fail, NOT a gate-bypassing force-fail; cumulative across retry attempts of the same turn). A trigger whose polarity the instance can't decide (per the abstract, checker-independent `live_decidable_polarities()`, a pure function of the criterion's own fields, paired with the checker's `live_verdict` override by lint rule CE025, a registry-based whole-tree check) is **inert by design** — one dataset-fanned YAML line serves both positive rows (pass/timeout live) and distractor rows (fail live). Verdicts **latch**: once a criterion decides, its `live_verdict` is never polled again. Stop rule is weighted, not strict-boolean: `run_limits.stop_early_gate_threshold` (default `1.0`, reproducing strict-AND behavior exactly) is the minimum weighted score (`Σ weight·score / Σ weight` over the armed subset) required to pass; a fail-stop fires once the armed set's **ceiling** (best case for everything still undecided) can no longer reach the threshold — so a low-weight fail or timeout that can't doom the gate is absorbed and the run continues — and is **deferred while any pass-capable armed criterion is undecided** (a distractor misfire never truncates a positive row's recall signal); a pass-stop fires once the `on_pass: stop` subset's **floor** (worst case) already meets the threshold, and is symmetrically **deferred while any pass-capable armed criterion outside the `on_pass: stop` subset is undecided** (so an early pass never freezes a sibling `on_pass: continue` criterion's signal out of the trajectory). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a kill-switched (`stop_early: false`) run. Driven by `orchestration/early_stop.py::EarlyStopWatcher` (built when `early_stop_active(task)`: ≥1 armed criterion, kill switch not thrown) through the agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. Gating is **FIRED-ONLY**: a run the watcher actually cut gates on the **armed subset** via the weighted `EvaluationResult.armed_criteria_passed`; a run that completes naturally — armed or not — gates strict-AND via `all_criteria_passed`, so adding a block never changes the verdict of a run it didn't cut. Note the gate keys on the watcher having FIRED (`result.early_stop is not None`), not on confirmed truncation — an agent that ignores `should_stop`, or a stop firing on the final message, still gates armed-only. Every resolution-time guardrail violation is a hard error at resolution (plan *and* run); the one load-time case — a `stop_early:` block on a non-live criterion — is a pydantic schema error at task load, which the run surface reports as a skipped task like any other malformed task. A runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo` (incl. `gate_threshold` at stop time), report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. No blocks anywhere ⇒ behavior byte-for-byte unchanged. diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 878e72a9..8c4454f3 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -74,7 +74,7 @@ share one implementation, so they cannot drift apart. Use it when something *else* owns the verdict — an external harness that builds its own container and runs its own tests — or to separate one expensive agent run from grading you want to iterate on afterwards. Grade the results later with -[`coder-eval evaluate`](#coder-eval-evaluate--test-criteria-without-an-agent). +[`coder-eval evaluate`](#coder-eval-evaluate--grade-without-running-an-agent). **Only the verdict is withheld, never the facts of the run.** A crash, timeout, or budget breach still reports `ERROR` / `TIMEOUT` / `TOKEN_BUDGET_EXCEEDED` and still @@ -105,23 +105,58 @@ Checks task syntax, required CLI tools, API keys, and schema validity without ex | --- | --- | | `--experiment, -e` | Experiment definition YAML to resolve variants against (default: `experiments/default.yaml`). | -### `coder-eval evaluate` — test criteria without an agent +### `coder-eval evaluate` — grade without running an agent + +Two shapes, told apart by whether the target holds a `task.json`: ```bash -coder-eval evaluate tasks/hello_date.yaml ./my_solution # evaluate a directory -coder-eval evaluate tasks/hello_date.yaml ./my_solution --preserve # keep the sandbox +# 1. Grade a directory against a task +coder-eval evaluate tasks/hello_date.yaml ./my_solution + +# 2. Re-grade a finished run — including one left NOT_GRADED by `execute` +coder-eval execute tasks/hello_date.yaml --run-dir ./r +coder-eval evaluate ./r/default/hello_date/00 +coder-eval aggregate ./r # run.json now reports the verdict ``` -Runs a task's success criteria against a directory without an agent — useful for -testing criterion definitions, validating task configs, or scoring code that was -already written. +**Run-directory mode** rebuilds the task from the run's own recorded +`task_config.resolved`, not by re-reading the YAML. That is what makes the grade +describe the run that happened: variant overrides, `-D` flags and dataset row +expansion are already baked into `resolved`, so re-loading the source would +silently grade a *different* task. The run's trajectory is restored too, so +criteria that read the agent's tool calls (`command_executed`, `skill_triggered`, +judges with trajectory) score exactly as they would have during the run. + +It writes the verdict back into the run's `task.json` and keeps the pre-grade +record beside it as `task.execute.json`. Writing back in place is what makes +`aggregate` free — no new flag, no second copy of the results. + +Passing a task file **over** a run directory re-grades it with different +criteria, reusing the trajectory and workspace of a run you already paid for: + +```bash +coder-eval evaluate tasks/hello_date.edited.yaml ./r/default/hello_date/00 +``` + +**In-place vs. copy.** The two-argument form copies your directory into a fresh +sandbox (criteria can mutate the target, and it is your own tree). Run-directory +mode grades **in place**, because copying filters build output — `node_modules`, +`dist`, `build`, `.venv`, `.git` are all on the default ignore list, so a +criterion like `test -f dist/bundle.js` would fail as a *copying artifact* +rather than as a verdict. Override either default with `--in-place` / `--copy`. | Flag | Description | | --- | --- | -| `--preserve / --no-preserve` | Preserve sandbox after evaluation (default: preserve) | -| `--run-dir` | Custom run directory (default: auto-generated timestamped dir in `runs/`). | +| `--workspace` | Grade this directory instead of the run's own artifacts (run-directory mode only). | +| `--in-place / --copy` | Grade where the files are, or copy first. Default: in-place for a run directory, copy for a plain work directory. | +| `--preserve / --no-preserve` | Preserve sandbox after evaluation (default: preserve). Ignored when grading in place — an adopted directory is never moved or deleted. | +| `--run-dir` | Where the graded `task.json` lands (default: auto-generated timestamped dir in `runs/`). | | `--verbose, -v` | DEBUG-level logging | +A re-grade refuses to run if the task's `reference:` directory changed since the +run (digest mismatch) — grading then would score the agent's old work against a +new answer key. + ### `coder-eval report` — view results ```bash diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index 1e62d886..cdba3fc5 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -1,6 +1,10 @@ -"""Evaluate command - run criteria against a directory without an agent.""" +"""Evaluate command - run criteria against a directory or re-grade a finished run.""" + +from __future__ import annotations import asyncio +import logging +from dataclasses import dataclass from pathlib import Path import typer @@ -11,6 +15,7 @@ EvaluationResult, FinalStatus, PreservationMode, + TaskDefinition, TemplateDirSource, parse_agent_config, ) @@ -18,21 +23,215 @@ from ..orchestrator import Orchestrator from ..sandbox import Sandbox from .console import console +from .evaluate_target import TASK_JSON, EvaluateMode, EvaluateTarget, EvaluateTargetError, resolve_evaluate_target from .run_helpers import prepare_run_directory +logger = logging.getLogger(__name__) + +# Where a run directory keeps the workspace the agent worked in. The extra +# segment is the task id: preservation writes `artifacts//...`. +ARTIFACTS_DIRNAME = "artifacts" + + +def _load_prior_result(run_dir: Path) -> EvaluationResult: + """Read a finished run's ``task.json``.""" + raw = (run_dir / TASK_JSON).read_text(encoding="utf-8") + try: + return EvaluationResult.model_validate_json(raw) + except ValueError as e: + raise typer.BadParameter(f"{run_dir / TASK_JSON} is not a readable EvaluationResult: {e}") from e + + +def _task_from_prior(prior: EvaluationResult, run_dir: Path) -> tuple[TaskDefinition, str]: + """Rebuild the executed task from the run's own recorded config. + + Rebuilding from ``task_config.resolved`` rather than re-reading the YAML is + what makes the grade describe the run that happened: ``resolved`` is the + post-merge definition, so variant overrides, ``-D`` flags and dataset row + expansion are all already baked in. Re-loading the source YAML would silently + grade a DIFFERENT task whenever any of those were used. + + Falls back to the source YAML only when ``resolved`` will not validate (a + schema change since the run), and says so loudly — a quiet fallback would + reintroduce exactly the drift above. + """ + record = prior.task_config + if record is None: + raise typer.BadParameter( + f"{run_dir / TASK_JSON} carries no task_config, so the executed task cannot be " + + "rebuilt. Pass the task file explicitly: coder-eval evaluate " + ) + try: + return TaskDefinition.model_validate(record.resolved), record.source_yaml + except ValueError as e: + if not record.source_file or not Path(record.source_file).is_file(): + raise typer.BadParameter( + f"The resolved task config in {run_dir / TASK_JSON} no longer validates ({e}), and " + + "its source YAML is unavailable. Pass the task file explicitly." + ) from e + console.print( + f"[yellow]⚠[/] The recorded resolved config does not validate ({e}); falling back to " + + f"{record.source_file}. Variant overrides, -D flags and dataset expansion from the " + + "original run are NOT reapplied, so this grade may not match what ran." + ) + return load_task(Path(record.source_file)) + + +def _default_workspace(run_dir: Path, prior: EvaluationResult) -> Path: + """Locate the workspace a finished run left behind. + + ``sandbox_path`` is authoritative when it still exists — it is where the run + actually worked. Otherwise fall back to the preserved artifacts tree, whose + single child is named for the task. + """ + if prior.sandbox_path: + recorded = Path(prior.sandbox_path) + if recorded.is_dir(): + return recorded + + artifacts = run_dir / ARTIFACTS_DIRNAME + if not artifacts.is_dir(): + raise typer.BadParameter( + f"No workspace to grade: {artifacts} does not exist and the recorded sandbox_path " + + f"({prior.sandbox_path or 'unset'}) is gone. The run was probably made with " + + "--preservation-mode NONE. Point at one explicitly with --workspace." + ) + children = [p for p in sorted(artifacts.iterdir()) if p.is_dir()] + # Preservation nests the workspace under the task id; a flat artifacts dir + # (no subdirectory) means the workspace IS artifacts/. + return children[0] if len(children) == 1 else artifacts + + +def _verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition) -> None: + """Refuse to grade when the reference tree changed since the run. + + ``reference_comparison`` and reference-carrying judges score against + ``task.reference.directory``. If it moved since the run, the re-grade would + silently measure the agent's old work against a new answer key. + """ + recorded = prior.environment_info.get("reference_digest") + if not isinstance(recorded, str) or task.reference is None: + return + from ..orchestration.evaluation import resolve_reference_dir + from ..path_utils import digest_tree + + resolved = resolve_reference_dir(task, None) + if resolved is None or not resolved.is_dir(): + return + if digest_tree(resolved) != recorded: + raise typer.BadParameter( + f"The reference directory {resolved} changed since this run was executed " + + "(digest mismatch). Grading now would score the agent's work against a " + + "different answer key. Restore the reference, or re-run the task." + ) + + +@dataclass(frozen=True) +class _ResolvedInputs: + """Everything the two positionals + ``--workspace`` decide, resolved once.""" + + target: EvaluateTarget + task: TaskDefinition + source_yaml: str + work_dir: Path + task_file: Path | None + prior: EvaluationResult | None + + +def _resolve_inputs(task_or_run_dir: Path, work_dir: Path | None, workspace: Path | None) -> _ResolvedInputs: + """Turn the CLI positionals into a task, a workspace, and (maybe) a prior run. + + Split out of the command because it is where both shapes converge: after this + the rest of ``evaluate`` is one code path regardless of which form was used. + """ + try: + target = resolve_evaluate_target(task_or_run_dir, work_dir) + except EvaluateTargetError as e: + raise typer.BadParameter(str(e)) from e + + if workspace is not None and target.mode is not EvaluateMode.RUN_DIR: + raise typer.BadParameter( + "--workspace applies to a run directory only; in the two-argument form the " + + "directory to grade is already the second argument." + ) + + prior: EvaluationResult | None = None + if target.mode is EvaluateMode.RUN_DIR: + prior = _load_prior_result(target.target) + if target.task_file is not None: + task, source_yaml = load_task(target.task_file) + console.print(f"[dim]Grading with {target.task_file} (overrides the run's recorded config).[/dim]") + else: + task, source_yaml = _task_from_prior(prior, target.target) + work_dir = workspace or _default_workspace(target.target, prior) + recorded_source = prior.task_config.source_file if prior.task_config else None + task_file = target.task_file or (Path(recorded_source) if recorded_source else None) + else: + assert target.task_file is not None # guaranteed by resolve_evaluate_target + task_file = target.task_file + try: + task, source_yaml = load_task(task_file) + except Exception as e: + console.print(f"[red]✗ Failed to load task:[/red] {e}") + raise typer.Exit(1) from e + work_dir = target.target + + if not work_dir.is_dir(): + console.print(f"[red]✗ Work directory is not a directory:[/red] {work_dir}") + raise typer.Exit(1) + + # Evaluate-only mode bypasses experiment resolution + CLI overrides, so + # `agent` may be None or `agent.type` may be unset for tasks that defer + # those to the experiment / CLI layers. The orchestrator only uses + # `agent.type` for result labeling here (no agent is created), so a + # default is safe. + if task.agent is None: + task.agent = parse_agent_config(type=AgentKind.CLAUDE_CODE) + elif task.agent.type is None: + task.agent = parse_agent_config(**{**task.agent.model_dump(exclude_unset=True), "type": AgentKind.CLAUDE_CODE}) + + if prior is not None: + _verify_reference_unchanged(prior, task) + + return _ResolvedInputs( + target=target, + task=task, + source_yaml=source_yaml, + work_dir=work_dir, + task_file=task_file, + prior=prior, + ) + + def evaluate_command( - task_file: Path = typer.Argument( # noqa: B008 + task_or_run_dir: Path = typer.Argument( # noqa: B008 ..., - help="Path to task YAML file", + metavar="[TASK_FILE] TARGET", + help="Task YAML file, or (when it is the only argument) a finished run directory.", exists=True, ), - work_dir: Path = typer.Argument( # noqa: B008 - ..., - help="Directory containing the code to evaluate", - exists=True, - file_okay=False, - dir_okay=True, + work_dir: Path | None = typer.Argument( # noqa: B008 + None, + metavar="", + help="Directory containing the code to evaluate. Omit when TASK_FILE is a run directory.", + ), + workspace: Path | None = typer.Option( # noqa: B008 + None, + "--workspace", + help=( + "Grade this directory instead of the run's own artifacts. Run-directory mode only (e.g. a verifier's /app)." + ), + ), + in_place: bool | None = typer.Option( + None, + "--in-place/--copy", + help=( + "Grade the workspace where it is, or copy it into a fresh sandbox first. " + "Default: in-place for a run directory, copy for a plain work directory. " + "Copying filters build output (node_modules, dist, build, .venv), so a " + "criterion that reads those needs --in-place." + ), ), verbose: bool = typer.Option( False, @@ -49,43 +248,75 @@ def evaluate_command( run_dir: Path | None = typer.Option( # noqa: B008 None, "--run-dir", - help="Custom run directory (default: auto-generated timestamped directory in runs/)", + help="Where the graded task.json lands (default: auto-generated timestamped directory in runs/)", ), ) -> None: - """Evaluate criteria against a directory without running an agent. + """Evaluate criteria against a directory, or re-grade a finished run. - Runs the success criteria defined in a task against a work directory. - Artifacts are saved to a run directory when --preserve is used. + Two shapes, told apart by whether the target holds a task.json: - Examples: + \b + Grade a directory against a task (no agent runs): coder-eval evaluate tasks/hello.yaml ./my_solution - coder-eval evaluate tasks/test.yaml /path/to/code --preserve - coder-eval evaluate tasks/test.yaml /path/to/code --run-dir ./my_eval_run + + \b + Re-grade a finished run — including one produced by `coder-eval execute`, + which leaves every task NOT_GRADED. The run's own task.json supplies the + resolved config AND the trajectory, so criteria that read the agent's tool + calls score exactly as they would have during the run: + coder-eval execute tasks/hello.yaml --run-dir ./r + coder-eval evaluate ./r/default/hello/00 + + \b + Iterate on criteria against a run you already paid for, by passing a task + file over a run directory (its trajectory and workspace are still used): + coder-eval evaluate tasks/hello.edited.yaml ./r/default/hello/00 """ - setup_logging(verbose=verbose) + run_evaluation( + task_or_run_dir=task_or_run_dir, + work_dir=work_dir, + workspace=workspace, + in_place=in_place, + verbose=verbose, + preserve=preserve, + run_dir=run_dir, + ) - console.print("\n[bold]Evaluating Criteria[/bold]\n") - try: - task, source_yaml = load_task(task_file) - except Exception as e: - console.print(f"[red]✗ Failed to load task:[/red] {e}") - raise typer.Exit(1) from e +def run_evaluation( + *, + task_or_run_dir: Path, + work_dir: Path | None = None, + workspace: Path | None = None, + in_place: bool | None = None, + verbose: bool = False, + preserve: bool = True, + run_dir: Path | None = None, +) -> None: + """The body of ``coder-eval evaluate``, with real Python defaults. - # Evaluate-only mode bypasses experiment resolution + CLI overrides, so - # `agent` may be None or `agent.type` may be unset for tasks that defer - # those to the experiment / CLI layers. The orchestrator only uses - # `agent.type` for result labeling here (no agent is created), so a - # default is safe. + Split from the Typer signature so it is directly callable: invoking a Typer + command function in-process hands every unspecified option an ``OptionInfo`` + sentinel rather than its default, which silently turns ``in_place=None`` into + a truthy object. Callers (tests, and any library use) call this instead. + """ + setup_logging(verbose=verbose) - if task.agent is None: - task.agent = parse_agent_config(type=AgentKind.CLAUDE_CODE) - elif task.agent.type is None: - task.agent = parse_agent_config(**{**task.agent.model_dump(exclude_unset=True), "type": AgentKind.CLAUDE_CODE}) + console.print("\n[bold]Evaluating Criteria[/bold]\n") - if not work_dir.is_dir(): - console.print(f"[red]✗ Work directory is not a directory:[/red] {work_dir}") - raise typer.Exit(1) + inputs = _resolve_inputs(task_or_run_dir, work_dir, workspace) + task = inputs.task + source_yaml = inputs.source_yaml + graded_dir = inputs.work_dir + task_file = inputs.task_file + prior = inputs.prior + target = inputs.target + + # In-place is the default for a run directory: that workspace is the run's + # own output and copying it would filter build artifacts out of the grade. + # A plain work directory defaults to copying, because criteria can mutate the + # target and it is the user's own tree. + grade_in_place = in_place if in_place is not None else (target.mode is EvaluateMode.RUN_DIR) try: prepared_run_dir = prepare_run_directory(run_dir) @@ -93,27 +324,38 @@ def evaluate_command( console.print(f"[red]✗ Failed to prepare run directory:[/red] {e}") raise typer.Exit(1) from e - # Build a sandbox pre-loaded with the work_dir contents, then run evaluate-only sandbox_config = task.sandbox.model_copy(deep=True) - template_source = TemplateDirSource(path=str(work_dir.resolve())) - if sandbox_config.template_sources: - sandbox_config.template_sources = [template_source, *sandbox_config.template_sources] - else: - sandbox_config.template_sources = [template_source] + if not grade_in_place: + # Copy path: preload the sandbox with the work dir as a template source. + template_source = TemplateDirSource(path=str(graded_dir.resolve())) + sandbox_config.template_sources = [template_source, *(sandbox_config.template_sources or [])] + # Grading never runs a container: the docker driver dispatches through + # DockerRunner, which needs an agent. Force tempdir so a task whose YAML says + # `driver: docker` is still gradeable on the host. + sandbox_config = sandbox_config.model_copy(update={"driver": "tempdir"}) - task_dir = task_file.parent.resolve() + task_dir = task_file.parent.resolve() if task_file is not None else None sandbox = Sandbox(sandbox_config, task_id=task.task_id, task_dir=task_dir) async def _setup_and_run() -> EvaluationResult: - await asyncio.to_thread(sandbox.setup) + if grade_in_place: + await asyncio.to_thread(sandbox.adopt, graded_dir) + else: + await asyncio.to_thread(sandbox.setup) orchestrator = Orchestrator( task=task, run_dir=prepared_run_dir, - preservation_mode=PreservationMode.MOVE_ON_WRITE if preserve else PreservationMode.NONE, + # An adopted directory is the caller's; never move or delete it. + preservation_mode=( + PreservationMode.NONE + if grade_in_place + else (PreservationMode.MOVE_ON_WRITE if preserve else PreservationMode.NONE) + ), task_file=task_file, sandbox=sandbox, - variant_id="evaluate", + variant_id=prior.variant_id if prior is not None else "evaluate", source_yaml=source_yaml, + prior_result=prior, ) return await orchestrator.run() @@ -162,6 +404,13 @@ async def _setup_and_run() -> EvaluationResult: if result.sandbox_path: console.print(f"[dim]Artifacts: {result.sandbox_path}[/dim]") + if prior is not None: + console.print( + f"[dim]Re-graded {prior.final_status.value} → {result.final_status.value} " + + f"over {len(result.iterations)} recorded turn(s).[/dim]" + ) + _write_back(target.target, result) + if result.final_status == FinalStatus.ERROR: console.print(f"\n[red]✗ Evaluation error: {result.error_message}[/red]") raise typer.Exit(1) @@ -171,3 +420,31 @@ async def _setup_and_run() -> EvaluationResult: else: console.print(f"\n[red]{failed} criterion/criteria failed.[/red]") raise typer.Exit(1) + + +def _write_back(run_dir: Path, result: EvaluationResult) -> None: + """Replace the graded run's ``task.json`` with the verdict, keeping a copy of the original. + + Updating in place is what makes the rest of the toolchain free: plain + ``coder-eval aggregate `` then rebuilds ``run.json`` from these rows + with no new code, and every report and evalboard view reads the graded row. + + The pre-grade original is kept alongside as ``task.execute.json`` so the + ungraded record is auditable — the write is not a silent overwrite of the + only evidence that the run was executed separately. + """ + target = run_dir / TASK_JSON + backup = run_dir / "task.execute.json" + try: + if not backup.exists(): + backup.write_text(target.read_text(encoding="utf-8"), encoding="utf-8") + target.write_text(result.model_dump_json(indent=2), encoding="utf-8") + except OSError as e: + # Never fail the grade over the write-back: the verdict was computed and + # already printed, and the fresh run dir holds its own task.json. + console.print(f"[yellow]⚠[/] Could not update {target}: {e}") + return + console.print( + f"[dim]Updated {target} (original kept as {backup.name}); " + + "run `coder-eval aggregate` to refresh run.json.[/dim]" + ) diff --git a/src/coder_eval/cli/evaluate_target.py b/src/coder_eval/cli/evaluate_target.py new file mode 100644 index 00000000..d8039a4e --- /dev/null +++ b/src/coder_eval/cli/evaluate_target.py @@ -0,0 +1,99 @@ +"""Shape detection for ``coder-eval evaluate``'s positional arguments. + +``evaluate`` accepts two shapes that look alike on the command line: + + coder-eval evaluate tasks/hello.yaml ./my_solution # grade a directory + coder-eval evaluate runs/latest/default/hello/00 # re-grade a finished run + +Both are "a task and a place", but the second carries its own task config and +trajectory inside ``task.json``, so nothing needs to be supplied twice. Rather +than adding a ``--run-dir-mode`` flag the caller has to remember, the shape is +detected from the target: a directory holding ``task.json`` is a run directory. + +The logic lives here, apart from the Typer command, because it is pure — it does +one ``is_file`` probe and otherwise just maps arguments to a decision — so it can +be tested exhaustively without building sandboxes or invoking a CLI runner. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path + + +TASK_JSON = "task.json" + + +class EvaluateMode(StrEnum): + """Which of the two shapes the caller asked for.""" + + WORK_DIR = "work_dir" + """Grade a plain directory against a task file. The original behavior.""" + + RUN_DIR = "run_dir" + """Re-grade a finished run: its task.json supplies config and trajectory.""" + + +@dataclass(frozen=True) +class EvaluateTarget: + """The resolved intent behind ``evaluate``'s positional arguments.""" + + mode: EvaluateMode + target: Path + """The run directory (RUN_DIR) or the directory to grade (WORK_DIR).""" + + task_file: Path | None + """Explicit task YAML. Required in WORK_DIR mode; an optional override in RUN_DIR mode.""" + + +class EvaluateTargetError(ValueError): + """The two positionals do not describe either supported shape.""" + + +def is_run_dir(path: Path) -> bool: + """Whether ``path`` is a finished task run directory (it holds ``task.json``).""" + return (path / TASK_JSON).is_file() + + +def resolve_evaluate_target(first: Path, second: Path | None) -> EvaluateTarget: + """Map ``evaluate``'s one-or-two positionals onto a mode + target. + + Args: + first: The first positional — a task file, or a run directory when it is + the only argument. + second: The second positional (the directory to grade), or None. + + Returns: + The resolved target. + + Raises: + EvaluateTargetError: If the arguments match neither shape. The message + always names what was passed and what to pass instead: a caller who + gets this wrong is one keystroke from the right command, and a bare + "invalid arguments" would not tell them which one. + """ + if second is None: + # One argument: only the run-dir shape is unambiguous. A lone task file + # names no place to grade, and a lone plain directory names no criteria. + if not first.is_dir(): + raise EvaluateTargetError( + f"{first} is not a directory. With a single argument, pass a finished run " + + f"directory (one containing {TASK_JSON}). To grade a directory against a " + + "task, pass both: coder-eval evaluate " + ) + if not is_run_dir(first): + raise EvaluateTargetError( + f"{first} holds no {TASK_JSON}, so it is not a run directory. Pass the task " + + f"file too: coder-eval evaluate {first}" + ) + return EvaluateTarget(mode=EvaluateMode.RUN_DIR, target=first, task_file=None) + + # Two arguments. The second is the place; the first is the task file. When + # that place turns out to be a run directory the caller is re-grading it with + # a DIFFERENT task file than the one it ran with — the "iterate on my + # criteria against an expensive run I already paid for" case, which is the + # main reason to keep `execute` and `evaluate` separate at all. Allow it, and + # let the caller be told which config won. + mode = EvaluateMode.RUN_DIR if second.is_dir() and is_run_dir(second) else EvaluateMode.WORK_DIR + return EvaluateTarget(mode=mode, target=second, task_file=first) diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 22092c17..0549e7b7 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -364,6 +364,7 @@ def __init__( replicate_index: int = 0, workspace_dir: Path | None = None, grade: bool = True, + prior_result: EvaluationResult | None = None, ): """Initialize the orchestrator. @@ -393,6 +394,11 @@ def __init__( deliberately NOT a task-config field — a task YAML must never be able to declare itself ungraded — so it arrives only from ``BatchRunConfig.grade``, never from the 5-layer merge or -D. + prior_result: A completed run's ``EvaluationResult`` to re-grade + (evaluate-only mode). Its trajectory and execution facts are + carried onto the fresh result so the grade describes the run that + actually happened instead of an empty one — see + ``_seed_from_prior_result`` for the field-by-field rationale. """ self.task = task self.run_dir = run_dir @@ -412,6 +418,7 @@ def __init__( self.config_lineage = config_lineage or {} self.replicate_index = replicate_index self.grade = grade + self.prior_result = prior_result # Derived paths self.report_path = self.run_dir / "task.json" @@ -540,6 +547,8 @@ async def run(self) -> EvaluationResult: environment_info=get_version_info(), ) + self._seed_from_prior_result() + # Calculate task log path task_log_file = task_log_path(self.run_dir) task_log_file.parent.mkdir(parents=True, exist_ok=True) # noqa: CE002 — mkdir on local FS is nanoseconds @@ -717,6 +726,51 @@ def _kill_agent_subprocess_sync() -> None: return self.result + def _seed_from_prior_result(self) -> None: + """Carry a completed run's execution facts onto this re-grade's result. + + ``execute`` then ``evaluate`` must equal a single ``run``. Everything + below is a fact the AGENT phase established that the grading phase cannot + re-derive; leaving any of them at their defaults would publish a row that + silently disagrees with the run it grades. + + Deliberately NOT carried: ``final_status``, ``weighted_score`` and + ``success_criteria_results`` — those are exactly what this pass recomputes + — and the timestamps/duration, which describe the grading pass. + """ + prior = self.prior_result + if prior is None or self.result is None: + return + + # The trajectory itself. Every derived figure in _finalize_result — + # token totals, cost, command_stats, model_used, assistant turns — + # recomputes from `iterations`, so seeding it reproduces them exactly. + self.result.iterations = list(prior.iterations) + # Evaluate-only hardcodes 1; a multi-turn run must not be reported as + # single-turn just because the re-grade ran once. + self.result.iteration_count = prior.iteration_count or len(prior.iterations) + + # LOAD-BEARING for the verdict: gate selection is FIRED-ONLY. When + # early_stop is not None the checker gates on the weighted ARMED subset + # instead of strict-AND over every criterion. Dropping it would re-grade a + # truncated trajectory under the full-run gate and flip the verdict. + self.result.early_stop = prior.early_stop + + # Execution facts that outlive the agent process. + self.result.max_turns_exhausted = prior.max_turns_exhausted + self.result.error_message = prior.error_message + self.result.error_details = prior.error_details + self.result.sdk_options = prior.sdk_options + + # environment_info: the prior run's capture describes the machine that + # RAN the task (installed_tools, api route, coder_eval version). Ours + # describes the machine grading it. Prior wins on conflict, and ours is + # preserved wholesale under `graded_by` rather than being interleaved — + # a report that shows the grader's tool versions as the run's is worse + # than one that shows neither. + graded_by = dict(self.result.environment_info) + self.result.environment_info = {**graded_by, **prior.environment_info, "graded_by": graded_by} + async def _run_evaluation_with_failure_evidence( self, *, @@ -1166,6 +1220,14 @@ def _join_litellm_actual_cost(self) -> None: """ if not (isinstance(self.route, LiteLLMRoute) and settings.litellm_cost_log and self.result is not None): return + if self.prior_result is not None: + # Re-grading someone else's trajectory. The join keys on THIS + # Orchestrator's per-attempt nonce, which the original turns were + # never tagged with, so it would match nothing and overwrite the + # already-corrected per-turn costs with a warning about a missing + # bill. The prior run's cost is the real one; leave it alone. + logger.debug("Re-grade of a prior trajectory: keeping its recorded cost, skipping the LiteLLM join.") + return try: applied = apply_actual_cost( self.result, @@ -1340,6 +1402,14 @@ async def _setup(self) -> None: self.sandbox.reference_dir = self._reference_dir self.result.sandbox_path = str(self.sandbox.sandbox_dir) + # PATH parity with the run being graded. _sync_sandbox_command_path_ + # with_agent recorded the agent's effective PATH; no agent runs here, + # so restore it explicitly or `run_command` criteria resolve binaries + # against ambient PATH and can disagree with the original verdict. + restored_path = self.result.environment_info.get("command_base_path") + if isinstance(restored_path, str) and restored_path: + self.sandbox.set_command_base_path(restored_path) + self._resolve_routes() self._record_route_environment_info() return @@ -1511,6 +1581,13 @@ def _sync_sandbox_command_path_with_agent(self) -> None: path = sdk_env.get("PATH") if isinstance(path, str) and path: self.sandbox.set_command_base_path(path) + # Persist it so a LATER detached grade (`coder-eval evaluate` over a + # finished run dir) can restore the same PATH. Without this the + # "evaluate-only mode" gap named above is permanent: the re-grade + # would resolve `run_command` criteria against ambient PATH and could + # reach a different verdict than the run it claims to be grading. + if self.result is not None: + self.result.environment_info["command_base_path"] = path def _eval_route_overrides(self) -> EvalRouteOverrides: """The ``(backend, model)`` pair from ``task.checker_context.api_route``, if any. @@ -1952,7 +2029,12 @@ async def _evaluation_loop(self) -> bool: "Criteria %s require agent execution; results may be incomplete with no agent", unsupported, ) - self.result.iteration_count = 1 + # A bare `evaluate ` has no trajectory, so one nominal + # iteration stands for the single grading pass. A re-grade seeded + # from a prior result already carries the real count (and the turns + # the trajectory-reading criteria need) — do not flatten it to 1. + if self.prior_result is None: + self.result.iteration_count = 1 # Load reference in evaluate-only mode too: judge criteria with # include_reference=true expect this populated even when no agent # runs. The agent-driven branch below has the same call. diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 55a24b5a..3fbead1b 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -266,6 +266,83 @@ def setup(self, target_dir: Path | None = None) -> Path: ) raise ValueError(f"Unsupported sandbox driver: {self.config.driver}") + def adopt(self, workspace: Path) -> Path: + """Use ``workspace`` **as** the sandbox, materializing nothing into it. + + The grade-in-place counterpart to :meth:`setup`. ``setup`` builds a + workspace: it copies template sources in, generates ``record_cli`` shims, + creates a venv, installs packages. ``adopt`` takes a workspace that + already exists — an ``execute`` run's artifacts, or a verifier's ``/app`` + — and only derives the *environment* the criteria need to run against it + (mock-dir ``+x``, venv discovery, the plugin-tools pin). + + Why not ``setup(target_dir=workspace)``: that already adopts a + caller-supplied directory and sets ``_cleanup_on_exit=False``, but it + then runs ``_setup_template()``, which would write over the very files + it was asked to grade. + + In-place is more CORRECT here, not merely faster: + + * ``_setup_template`` filters what it copies through + ``_should_ignore_template_file`` — ``node_modules``, ``dist``, + ``build``, ``venv``, ``.git`` and friends are dropped. A criterion like + ``test -f dist/bundle.js`` therefore fails as a *copying artifact* + rather than as a verdict on the agent's work. + * ``run_command`` criteria execute with ``cwd = sandbox_dir``, so on the + copy path they see the copy's paths, not the ones the agent worked at. + * Copying a real workspace costs minutes. + + The caller keeps ownership: ``_cleanup_on_exit`` stays False, so + ``cleanup()`` never deletes an adopted directory. Criteria CAN still + mutate it (a ``run_command`` that writes), which is why the copy path + remains the default for a bare user-supplied work dir. + + Args: + workspace: An existing directory to grade in place. + + Returns: + Path to the sandbox directory (``workspace``). + + Raises: + RuntimeError: If the driver is ``docker`` (a container workspace is + not reachable from the host), or ``workspace`` is not a directory. + """ + if self.config.driver == "docker": + raise RuntimeError( + "Sandbox.adopt() is host-side only; a driver='docker' workspace lives inside " + + "the container. Grade it from within the container, or copy it out first." + ) + if not workspace.is_dir(): + raise RuntimeError(f"Cannot adopt {workspace}: not an existing directory") + + self.sandbox_dir = workspace.resolve() + # Never flipped True: an adopted directory belongs to the caller. + self._cleanup_on_exit = False + + # Only NON-materializing steps below. Deliberately skipped, and why: + # _setup_template would overwrite the workspace being graded + # _generate_cli_recorders writes shims into it + # _setup_virtualenv / + # _install_*_packages the execute phase already provisioned these; + # re-running mutates the graded tree + # _maybe_remediate_home_plugins_pollution + # destructive on $HOME, and it is remediation + # rather than derivation — the execute phase + # already ran it if it was enabled + self._prepare_mock_path_dirs() + + # Discover an existing venv instead of creating one, so `run_command` + # criteria get the same VIRTUAL_ENV/PATH the agent had. Absent venv -> + # None, exactly as for a task with no python config. + candidate = self.sandbox_dir / ".venv" + if candidate.is_dir(): + self.venv_dir = candidate + + self._check_parent_node_modules_contamination() + self._refresh_plugin_tools_dir() + + return self.sandbox_dir + def _setup_tempdir(self, target_dir: Path | None = None) -> Path: """Set up a sandbox directory. diff --git a/tests/test_evaluate_command.py b/tests/test_evaluate_command.py index 67a98046..06d26f3a 100644 --- a/tests/test_evaluate_command.py +++ b/tests/test_evaluate_command.py @@ -20,7 +20,7 @@ def test_evaluate_command_success(tmp_path): (work_dir / "app.py").write_text("print('hello')") # Import and run command - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() @@ -28,7 +28,7 @@ def test_evaluate_command_success(tmp_path): with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): # Should not raise - all criteria pass with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) assert exc_info.value.exit_code == 0 @@ -59,14 +59,14 @@ def test_evaluate_command_defaults_agent_type_when_missing(tmp_path): work_dir.mkdir() (work_dir / "app.py").write_text("print('hello')") - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) assert exc_info.value.exit_code == 0 @@ -85,7 +85,7 @@ def test_evaluate_command_maps_preserve_to_mode(tmp_path, preserve, expected_mod run_dir = tmp_path / "run" run_dir.mkdir() - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation captured: dict[str, PreservationMode] = {} @@ -105,7 +105,7 @@ async def run(self): patch("coder_eval.cli.evaluate_command.Orchestrator", _CapturingOrchestrator), pytest.raises(_StopError), ): - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir, preserve=preserve) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir, preserve=preserve) assert captured["mode"] == PreservationMode(expected_mode) @@ -119,7 +119,7 @@ def test_evaluate_command_failure(tmp_path): work_dir.mkdir() # Import and run command - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() @@ -127,7 +127,7 @@ def test_evaluate_command_failure(tmp_path): with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): # Should fail - file doesn't exist with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) assert exc_info.value.exit_code == 1 @@ -140,14 +140,14 @@ def test_evaluate_command_informational_criterion_does_not_fail_exit(tmp_path): work_dir.mkdir() (work_dir / "app.py").write_text("print('hello')") # gating criterion passes; missing.py absent - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) # The only gating criterion passed → exit 0, despite the weight=0 miss. assert exc_info.value.exit_code == 0 @@ -163,14 +163,14 @@ def test_evaluate_command_invalid_task_file(tmp_path): work_dir.mkdir() # Import and run command - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) assert exc_info.value.exit_code == 1 @@ -183,14 +183,14 @@ def test_evaluate_command_invalid_work_dir(tmp_path): work_dir = tmp_path / "nonexistent" # Import and run command - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) assert exc_info.value.exit_code == 1 @@ -205,14 +205,14 @@ def test_evaluate_command_multiple_criteria(tmp_path): (work_dir / "app.py").write_text("print('hello')") # Import and run command - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) # All 3 criteria should pass assert exc_info.value.exit_code == 0 diff --git a/tests/test_evaluate_target.py b/tests/test_evaluate_target.py new file mode 100644 index 00000000..1b908419 --- /dev/null +++ b/tests/test_evaluate_target.py @@ -0,0 +1,102 @@ +"""`resolve_evaluate_target` — the shape detection behind `coder-eval evaluate`. + +Pure and exhaustively testable by design: the command accepts two forms that +look alike on the command line, and picking the wrong one silently grades the +wrong thing. Every combination of (one arg / two args) x (run dir / plain dir / +file / missing) is enumerated here. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from coder_eval.cli.evaluate_target import ( + EvaluateMode, + EvaluateTargetError, + is_run_dir, + resolve_evaluate_target, +) + + +def _run_dir(tmp_path: Path, name: str = "run") -> Path: + d = tmp_path / name + d.mkdir() + (d / "task.json").write_text(json.dumps({"task_id": "t"}), encoding="utf-8") + return d + + +def _plain_dir(tmp_path: Path, name: str = "work") -> Path: + d = tmp_path / name + d.mkdir() + return d + + +def _file(tmp_path: Path, name: str = "task.yaml") -> Path: + f = tmp_path / name + f.write_text("task_id: t", encoding="utf-8") + return f + + +def test_is_run_dir_keys_on_task_json(tmp_path: Path) -> None: + assert is_run_dir(_run_dir(tmp_path)) + assert not is_run_dir(_plain_dir(tmp_path)) + + +# --- one argument --------------------------------------------------------- + + +def test_lone_run_dir_is_run_dir_mode(tmp_path: Path) -> None: + run = _run_dir(tmp_path) + resolved = resolve_evaluate_target(run, None) + assert resolved.mode is EvaluateMode.RUN_DIR + assert resolved.target == run + assert resolved.task_file is None, "a run dir carries its own config; nothing to supply" + + +def test_lone_task_file_is_rejected_with_the_fix(tmp_path: Path) -> None: + """A task file alone names no place to grade.""" + with pytest.raises(EvaluateTargetError, match="not a directory"): + resolve_evaluate_target(_file(tmp_path), None) + + +def test_lone_plain_dir_is_rejected_with_the_fix(tmp_path: Path) -> None: + """A plain directory alone names no criteria.""" + plain = _plain_dir(tmp_path) + with pytest.raises(EvaluateTargetError) as exc: + resolve_evaluate_target(plain, None) + # The message must name the missing piece AND the corrected command — a + # caller here is one argument away from the right invocation. + assert "task.json" in str(exc.value) + assert "" in str(exc.value) + + +# --- two arguments -------------------------------------------------------- + + +def test_task_file_plus_plain_dir_is_the_original_form(tmp_path: Path) -> None: + """The pre-existing shape must keep resolving exactly as before.""" + task, work = _file(tmp_path), _plain_dir(tmp_path) + resolved = resolve_evaluate_target(task, work) + assert resolved.mode is EvaluateMode.WORK_DIR + assert resolved.target == work + assert resolved.task_file == task + + +def test_task_file_plus_run_dir_re_grades_with_the_given_task(tmp_path: Path) -> None: + """Iterating on criteria against a run you already paid for: the run supplies + the trajectory and workspace, the explicit file supplies the criteria.""" + task, run = _file(tmp_path), _run_dir(tmp_path) + resolved = resolve_evaluate_target(task, run) + assert resolved.mode is EvaluateMode.RUN_DIR + assert resolved.target == run + assert resolved.task_file == task, "the override must survive; it is the whole point of this form" + + +def test_a_nonexistent_second_arg_stays_work_dir_mode(tmp_path: Path) -> None: + """Shape detection must not invent run-dir mode for a path that isn't there; + the command reports the missing directory itself, with a clearer message.""" + resolved = resolve_evaluate_target(_file(tmp_path), tmp_path / "nope") + assert resolved.mode is EvaluateMode.WORK_DIR diff --git a/tests/test_execute_evaluate_loop.py b/tests/test_execute_evaluate_loop.py new file mode 100644 index 00000000..c9c7264b --- /dev/null +++ b/tests/test_execute_evaluate_loop.py @@ -0,0 +1,147 @@ +"""The `execute` -> `evaluate` -> `aggregate` loop. + +`coder-eval execute` withholds the verdict; `coder-eval evaluate ` +supplies it later. The pair only earns its keep if it ends up where a single +`coder-eval run` would have: same status, same score, same criteria, and a +`run.json` the rest of the toolchain can read. + +Everything here runs against the agentless task — deterministic, no API key. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest +from typer.testing import CliRunner + +from coder_eval.cli import app +from coder_eval.models import FinalStatus + + +runner = CliRunner() + +AGENTLESS_TASK = Path("tasks/agentless_smoke_test.yaml") + +pytestmark = pytest.mark.skipif( + not AGENTLESS_TASK.is_file(), reason="needs a source checkout (tasks/ is not in the wheel)" +) + + +def _task_dir(run_dir: Path) -> Path: + matches = sorted(p.parent for p in run_dir.glob("**/task.json")) + assert len(matches) == 1, f"expected exactly one task.json under {run_dir}, got {matches}" + return matches[0] + + +def _row(task_dir: Path, name: str = "task.json") -> dict[str, Any]: + return json.loads((task_dir / name).read_text(encoding="utf-8")) + + +def _invoke(args: list[str]) -> Any: + result = runner.invoke(app, args) + assert result.exit_code == 0, f"{args} failed:\n{result.output}" + return result + + +def test_execute_then_evaluate_reaches_the_same_verdict_as_run(tmp_path: Path) -> None: + """The headline guarantee, asserted against a real `run` rather than a + hardcoded expectation — so a change that breaks BOTH paths still fails.""" + direct = tmp_path / "direct" + _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(direct)]) + expected = _row(_task_dir(direct)) + + split = tmp_path / "split" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(split)]) + _invoke(["evaluate", str(_task_dir(split))]) + actual = _row(_task_dir(split)) + + assert expected["final_status"] == FinalStatus.SUCCESS.value, "the fixture must actually pass under `run`" + assert actual["final_status"] == expected["final_status"] + assert actual["weighted_score"] == expected["weighted_score"] + assert [c["criterion_type"] for c in actual["success_criteria_results"]] == [ + c["criterion_type"] for c in expected["success_criteria_results"] + ] + assert [c["score"] for c in actual["success_criteria_results"]] == [ + c["score"] for c in expected["success_criteria_results"] + ] + + +def test_evaluate_upgrades_the_row_in_place_and_keeps_the_original(tmp_path: Path) -> None: + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + assert _row(task_dir)["final_status"] == FinalStatus.NOT_GRADED.value + + _invoke(["evaluate", str(task_dir)]) + + assert _row(task_dir)["final_status"] == FinalStatus.SUCCESS.value + # The pre-grade record survives, so "this run was executed separately" stays + # auditable rather than being silently overwritten. + assert _row(task_dir, "task.execute.json")["final_status"] == FinalStatus.NOT_GRADED.value + + +def test_aggregate_rebuilds_a_graded_run_json_with_no_extra_step(tmp_path: Path) -> None: + """Grading in place is what makes the rest of the toolchain free: the + existing `aggregate` command sees the upgraded rows with no new code.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + assert json.loads((run_dir / "run.json").read_text(encoding="utf-8"))["tasks_not_graded"] == 1 + + _invoke(["evaluate", str(_task_dir(run_dir))]) + _invoke(["aggregate", str(run_dir)]) + + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert summary["tasks_not_graded"] == 0 + assert summary["tasks_succeeded"] == 1 + assert summary["pass_rate"] == 1.0 + + +def test_re_grade_carries_the_trajectory_not_an_empty_one(tmp_path: Path) -> None: + """Criteria that read the agent's tool calls (command_executed, + skill_triggered, judges with trajectory) score off `iterations`. A re-grade + that dropped them would silently fail every such criterion.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + executed = _row(task_dir) + + _invoke(["evaluate", str(task_dir)]) + graded = _row(task_dir) + + assert len(graded["iterations"]) == len(executed["iterations"]) + assert graded["iteration_count"] == executed["iteration_count"] + + +def test_evaluate_does_not_move_or_delete_the_graded_workspace(tmp_path: Path) -> None: + """Run-dir mode adopts the workspace; the caller keeps ownership.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + proof = sorted(run_dir.glob("**/artifacts/**/proof.txt")) + assert proof, "fixture precondition: execute preserved a workspace" + + _invoke(["evaluate", str(_task_dir(run_dir))]) + + assert proof[0].is_file(), "the adopted workspace was moved or deleted" + + +def test_evaluate_still_grades_a_plain_directory(tmp_path: Path) -> None: + """The original two-argument form must keep working unchanged.""" + work = tmp_path / "work" + work.mkdir() + (work / "proof.txt").write_text("coder-eval-ran-without-a-coder", encoding="utf-8") + + result = runner.invoke(app, ["evaluate", str(AGENTLESS_TASK), str(work)]) + + assert result.exit_code == 0, result.output + assert "All criteria passed" in result.output + + +def test_evaluate_rejects_workspace_flag_outside_run_dir_mode(tmp_path: Path) -> None: + work = tmp_path / "work" + work.mkdir() + result = runner.invoke(app, ["evaluate", str(AGENTLESS_TASK), str(work), "--workspace", str(work)]) + assert result.exit_code != 0 + assert "run directory only" in result.output diff --git a/tests/test_litellm_cost.py b/tests/test_litellm_cost.py index ea23c8c7..ce40a43b 100644 --- a/tests/test_litellm_cost.py +++ b/tests/test_litellm_cost.py @@ -257,6 +257,8 @@ def test_joins_on_litellm_route(self, tmp_path, monkeypatch): _cost_correlation_run_id=run_id, _cost_attempt_nonce="att1", _log_task_id="calc", + # A live run, not a re-grade: the join applies (see _join_litellm_actual_cost). + prior_result=None, ) orch_mod.Orchestrator._join_litellm_actual_cost(fake) assert fake.result.iterations[0].token_usage.total_cost_usd == 0.09 # static 0.5 overridden @@ -270,6 +272,8 @@ def test_join_never_raises_on_bad_log(self, tmp_path, monkeypatch): _cost_correlation_run_id="R", _cost_attempt_nonce="att1", _log_task_id="calc", + # A live run, not a re-grade: the join applies (see _join_litellm_actual_cost). + prior_result=None, ) orch_mod.Orchestrator._join_litellm_actual_cost(fake) # missing file → no-op, no raise assert fake.result.iterations[0].token_usage.total_cost_usd == 0.5 @@ -294,6 +298,8 @@ def test_run_total_rederives_from_actual_after_join(self, tmp_path, monkeypatch) _cost_correlation_run_id=run_id, _cost_attempt_nonce="att1", _log_task_id="calc", + # A live run, not a re-grade: the join applies (see _join_litellm_actual_cost). + prior_result=None, ) orch_mod.Orchestrator._join_litellm_actual_cost(fake) orch_mod.Orchestrator._aggregate_token_usage(fake) diff --git a/tests/test_sandbox_adopt.py b/tests/test_sandbox_adopt.py new file mode 100644 index 00000000..39fafd9b --- /dev/null +++ b/tests/test_sandbox_adopt.py @@ -0,0 +1,108 @@ +"""`Sandbox.adopt` — grade a workspace in place instead of copying it. + +The behavior that matters is what adopt does NOT do. `setup()` materializes a +workspace (copies templates in, writes shims, builds a venv); `adopt()` takes one +that already exists and only derives the environment around it. A regression that +made adopt materialize anything would overwrite the very files being graded, and +would do so silently — the criteria would still run, just against different +content. So each assertion below pins one thing staying untouched. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from coder_eval.models import SandboxConfig +from coder_eval.sandbox import Sandbox + + +def _workspace(tmp_path: Path) -> Path: + """A workspace shaped like an agent's output: real files plus build output.""" + ws = tmp_path / "ws" + (ws / "node_modules" / "pkg").mkdir(parents=True) + (ws / "dist").mkdir() + (ws / "src.py").write_text("print('hi')", encoding="utf-8") + (ws / "node_modules" / "pkg" / "index.js").write_text("module.exports=1", encoding="utf-8") + (ws / "dist" / "bundle.js").write_text("bundled", encoding="utf-8") + return ws + + +def _sandbox(**cfg: object) -> Sandbox: + return Sandbox(SandboxConfig(**cfg), task_id="t") # type: ignore[arg-type] + + +def test_adopt_uses_the_directory_itself(tmp_path: Path) -> None: + ws = _workspace(tmp_path) + sandbox = _sandbox() + assert sandbox.adopt(ws) == ws.resolve() + assert sandbox.sandbox_dir == ws.resolve(), "adopt must not create a copy" + + +def test_adopt_exposes_files_the_copy_path_filters_out(tmp_path: Path) -> None: + """The bug this fixes. `_should_ignore_template_file` drops node_modules, + dist, build and .venv, so on the copy path a criterion like + `test -f dist/bundle.js` fails as a COPYING artifact rather than as a + verdict on the agent's work.""" + ws = _workspace(tmp_path) + sandbox = _sandbox() + sandbox.adopt(ws) + assert sandbox.sandbox_dir is not None + assert (sandbox.sandbox_dir / "dist" / "bundle.js").is_file() + assert (sandbox.sandbox_dir / "node_modules" / "pkg" / "index.js").is_file() + + +def test_adopt_writes_nothing_into_the_workspace(tmp_path: Path) -> None: + """No shims, no venv, no template files — the graded tree is exactly as found.""" + ws = _workspace(tmp_path) + before = {p.relative_to(ws) for p in ws.rglob("*")} + _sandbox().adopt(ws) + assert {p.relative_to(ws) for p in ws.rglob("*")} == before + + +def test_adopt_never_owns_the_directory(tmp_path: Path) -> None: + """cleanup() must not delete a directory the caller handed us.""" + ws = _workspace(tmp_path) + sandbox = _sandbox() + sandbox.adopt(ws) + assert sandbox.is_persistent + sandbox.cleanup() + assert ws.is_dir(), "cleanup deleted an adopted workspace" + assert (ws / "src.py").is_file() + + +def test_adopt_discovers_an_existing_venv_without_creating_one(tmp_path: Path) -> None: + """The execute phase already built the venv; re-creating it would mutate the + graded tree. Discovery keeps `run_command` criteria on the agent's PATH.""" + ws = _workspace(tmp_path) + (ws / ".venv" / "bin").mkdir(parents=True) + sandbox = _sandbox(python={"env_packages": []}) + sandbox.adopt(ws) + assert sandbox.venv_dir == ws.resolve() / ".venv" + + +def test_adopt_leaves_venv_unset_when_there_is_none(tmp_path: Path) -> None: + sandbox = _sandbox(python={"env_packages": []}) + sandbox.adopt(_workspace(tmp_path)) + assert sandbox.venv_dir is None + + +def test_adopt_rejects_the_docker_driver(tmp_path: Path) -> None: + """A container workspace is not reachable from the host, so adopting one + would silently grade whatever happens to sit at that host path.""" + sandbox = _sandbox(driver="docker") + with pytest.raises(RuntimeError, match="host-side only"): + sandbox.adopt(_workspace(tmp_path)) + + +def test_adopt_rejects_a_missing_directory(tmp_path: Path) -> None: + with pytest.raises(RuntimeError, match="not an existing directory"): + _sandbox().adopt(tmp_path / "nope") + + +def test_adopt_rejects_a_file(tmp_path: Path) -> None: + f = tmp_path / "a.txt" + f.write_text("x", encoding="utf-8") + with pytest.raises(RuntimeError, match="not an existing directory"): + _sandbox().adopt(f) From 7c13418ca113d46bda3dc2ad34512ba0aac71121 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Thu, 3 Sep 2026 14:16:03 -0700 Subject: [PATCH 03/11] feat(cli): make --resume distinguish "executed" from "graded" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--resume` decided a task was finished by asking "does task.json carry any final_status". NOT_GRADED is a final status, so `run --resume` over a run produced by `coder-eval execute` reported the tasks already complete, graded nothing, and exited 0: after execute: NOT_GRADED $ coder-eval run --run-dir tmp/res --resume ↻ Resume: 1 task(s) already complete, running 0 remaining Results: 1/1 executed, not graded real exit code: 0 after run --resume: NOT_GRADED ## "Finished" is relative to the resuming command `partition_for_resume(tasks, *, grade)` now returns a four-way `ResumePartition` (to_run / to_grade / prior_results / prior_resolved). A NOT_GRADED row owes `execute` nothing — it finished executing — but owes `run` a grade. Under grade=True those rows route to `to_grade`, where the criteria run against the trajectory and workspace already on disk instead of paying for the agent a second time. That reuse is the entire reason `execute` and `run` are separate commands. The carve-out is ONLY for NOT_GRADED. FAILURE and ERROR stay complete under both commands — resume has never retried failures (delete a task's task.json to force that) — and a parametrized test pins that so the carve-out cannot grow into a general "retry bad rows" rule. `clear_rerun_artifacts` skips `to_grade`, whose artifacts are the very thing being graded. A per-task grading failure is warned and folded back in with its ORIGINAL ungraded result, so one bad row neither aborts the resume nor vanishes from run.json — it stays visible as tasks_not_graded. `grade` joins `_FINGERPRINT_DIFF_EXEMPT`: execute → run --resume is a supported flow, not config drift, and the warning's "already-finalized tasks keep their original-config results" text is actively wrong for it (those rows are re-graded with the current config, which is the point). `execute --resume` is consequently supported and no longer refused. ## One implementation, not two `orchestration/regrade.py` now holds the re-grading core, shared by the resume path and `evaluate`'s run-dir mode. Two copies of "how to re-grade" would drift into two different verdicts for the same run. It raises a plain `RegradeError` that the CLI wraps, since orchestration/ must not import the CLI layer (CE004). ## Fidelity fix caught by writing the test A re-graded row was reporting the GRADING pass's clock. A 10-minute agent run re-graded in 2 seconds would record 2 seconds — and duration_seconds feeds VariantAggregate.average_duration, the report tables and the evalboard, so harness-vs-harness comparisons would have been quietly wrong. A task row describes the TASK, so it now keeps the agent run's `started_at` and `duration_seconds`. The grading pass's own cost is preserved separately as `environment_info["grading_duration_seconds"]` rather than discarded, so a slow judge stays visible. ## Verification `make verify` green (4612 passed, 92.07%). End-to-end: `run --resume` grades what execute left (NOT_GRADED → SUCCESS, pass_rate 1.0) while reporting "running 0 remaining", so the agent demonstrably did not re-run; the trajectory, started_at and duration_seconds all survive; task.execute.json is preserved by this path too; `execute --resume` treats the row as done; and no config-drift warning is emitted. Unit: the four-way partition under both grade values, and the failure-retry guard. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 6 +- docs/USER_GUIDE.md | 39 ++++- src/coder_eval/cli/evaluate_command.py | 132 ++++----------- src/coder_eval/cli/execute_command.py | 30 ++-- src/coder_eval/cli/run_command.py | 76 ++++++++- src/coder_eval/orchestration/batch.py | 61 +++++-- src/coder_eval/orchestration/regrade.py | 205 ++++++++++++++++++++++++ src/coder_eval/orchestrator.py | 18 ++- tests/test_execute_command.py | 1 - tests/test_execute_evaluate_loop.py | 68 ++++++++ tests/test_resume.py | 73 ++++++++- 11 files changed, 571 insertions(+), 138 deletions(-) create mode 100644 src/coder_eval/orchestration/regrade.py diff --git a/CLAUDE.md b/CLAUDE.md index 20f8f5e0..f4e4642e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -80,10 +80,11 @@ coder_eval/ │ └── timeout.py # Timeout handling (TurnTimeoutError carries optional partial TurnRecord) │ ├── orchestration/ # Batch execution utilities -│ ├── batch.py # Parallel task execution (run_batch + run_batch_resolved) +│ ├── batch.py # Parallel task execution (run_batch) + partition_for_resume/ResumePartition │ ├── config.py # Batch run configuration │ ├── early_stop.py # validate_early_stop guardrails + EarlyStopWatcher (armed live-verdict observer) │ ├── evaluation.py # Reference dir resolution + per-run private staging +│ ├── regrade.py # Grade an already-executed run in place — shared by `evaluate ` and `run --resume` │ ├── experiment.py # ExperimentRunner, resolve_task_for_variant, load_experiment │ └── task_loader.py # YAML task loading │ @@ -149,8 +150,9 @@ action.yml # Published composite GitHub Action (coder-ev - **Reference solutions are directory-only, and shielded (partially) from the agent**: `task.reference` is a single required `directory:` (relative to the task YAML) — the inline `code:` / single-file `file:` forms are gone, because a directory is the only shape that can be permission-gated as a unit; a `model_validator(mode="before")` gives the removed forms a migration error. The orchestrator stages a **per-run private copy** (`orchestration/evaluation.py::stage_reference_dir`, symlinks stripped) into a tempdir, removed in `_cleanup` via `path_utils.rmtree_restrictive` (keyed on `_reference_staging_root`, recorded BEFORE the copy so a failed copy still cleans up; `rmtree(ignore_errors=True)` silently declines on a tree left at 000) and deliberately never preserved into `run_dir/artifacts`. That copy is held at mode `000` for the whole of every `agent.communicate` call via **`Sandbox.set_permissions`**, the driver-aware wrapper over `fs_permissions.py::set_permissions`. Windows **stack**: exiting restores the *enclosing* window's mode, only the outermost exit restores the pre-window mode — that is what makes a mid-turn re-grant (`mode=READ_ONLY_MODE`) expressible, and it covers two windows at the same mode so no refcount is needed. The window is enforced **only inside a docker container** (`Sandbox.enforces_permission_windows`) and is a no-op on the host, where the agent shares our uid. **That gate keys on the `CODER_EVAL_IN_CONTAINER` env var, NOT `sandbox.driver`** — `run_task_internal_command` rewrites `driver: docker` → `tempdir` before building the in-container Orchestrator, so a driver-based gate would silently disable the anti-cheat on exactly the path that needs it (regression-guarded by `TestSandboxDriverGate`); `resolve_reference_dir` gates its `/work/references` branch on the same var for the same reason. The task directory is **not** shielded (`:ro` mount → EROFS, and the same YAML is readable at `/work/input`). Criteria address reference files with the `$REFERENCE_DIR` token (same resolver as `$TASK_DIR`) and the `REFERENCE_DIR` env var for `run_command`; `reference_comparison` names one file via `reference_file`. Docker mounts a throwaway **read-write** copy at `/work/references` (a `:ro` mount cannot be chmod'd — EROFS), masks the in-task-dir original with an empty tmpfs, and drops `DAC_OVERRIDE`/`DAC_READ_SEARCH`. `FOWNER`/`CHOWN` are deliberately **NOT** dropped: the in-container orchestrator that applies the window is the same root process with the same caps, so dropping `FOWNER` breaks *the harness's own* chmod wherever the bind mount preserves a non-root owner (native Linux — verified: `chmod: Operation not permitted`), i.e. exactly where the drop would otherwise bite. A window that cannot be applied is now a hard error, not a warning: `Sandbox.set_permissions` passes `strict=True` whenever it enforces, so an unprotected run fails instead of producing a normal-looking score. **KNOWN GAP — this is defense-in-depth, not a boundary**: (a) `chmod(2)` is gated on owner-or-`CAP_FOWNER` and the container runs as root owning the copy, so a deliberate `chmod 755 /work/references` restores access; (b) the window spans `agent.communicate` only, and nothing reaps agent child processes at turn end, so a backgrounded read loop succeeds once the window closes. The **write** half of (b) is closed — `path_utils.digest_tree` hashes the tree at staging and `Orchestrator._verify_reference_integrity` re-checks before grading, raising `ReferenceTamperedError` (→ `FinalStatus.ERROR`) on a mismatch so an agent cannot overwrite the reference to drive `reference_comparison` to 1.0. Passive reads are blocked; an adversarial agent is not. Full containment requires running the agent as a non-root uid AND holding the window for the agent's whole lifetime — follow-up. `tasks/anti_cheat_reference` probes the passive-read half. - **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex and Antigravity (both run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced), and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. Full table + rationale: docs/agents/HARNESS_PARITY.md. - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly. -- **Execute vs. run (the grading switch)**: `coder-eval execute` is `coder-eval run` with grading removed — the agent runs and the full trajectory is captured, but no criterion is checked, `weighted_score` is `None` (never `0.0`, which would be indistinguishable from "graded and scored zero"), and the row finalizes as **`FinalStatus.NOT_GRADED`**, whose `category` is a **fourth** bucket, `"ungraded"`. Ungraded rows leave BOTH sides of every rate: `RunSummary.pass_rate` / `error_share` and `VariantAggregate.pass_rate` divide by `tasks_graded` (`tasks_run - tasks_not_graded`), and `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter. **Only SUCCESS/FAILURE collapse into it** — `ERROR`, `TIMEOUT`, `BUILD_FAILED`, `MAX_TURNS_EXHAUSTED` and the budget stops are facts about the *run*, not about grading, and still apply (so `execute` still exits non-zero on a crash). The switch is `BatchRunConfig.grade` → `Orchestrator(grade=...)` → the **four** grading call sites (single-shot, evaluate-only, the simulation dialog check, and post-failure diagnostics); it crosses the docker boundary in `context.json` (defaulting to `True` in-container, so a host predating `execute` keeps grading). It is **deliberately not a task-config field** — no 5-layer merge, no `-D` path — because a task YAML must never declare itself ungraded; only the invoking command decides. `run` and `execute` share one body (`run_command.run_pipeline`) and differ solely in that flag, so there is no third code path. Three things are refused rather than degraded: `--junit-xml` (a report of verdicts, and there are none — though `reports_junit` still emits `` for an ungraded row it encounters), `--resume` (`partition_for_resume` treats "has any final status" as finalized, so a `NOT_GRADED` row would be *skipped* by a later `run --resume` instead of graded), and simulation tasks (their turn-continuation logic reads criteria results, so an ungraded dialog would silently change its own stopping behavior). `stop_early:` blocks are inert under `execute` for the same reason the kill switch exists: the full trajectory is the deliverable. Motivating consumer: an external harness (Harbor / Terminal-Bench 2.0) that builds its own container, calls coder-eval as the agent, and grades with its own tests. +- **Execute vs. run (the grading switch)**: `coder-eval execute` is `coder-eval run` with grading removed — the agent runs and the full trajectory is captured, but no criterion is checked, `weighted_score` is `None` (never `0.0`, which would be indistinguishable from "graded and scored zero"), and the row finalizes as **`FinalStatus.NOT_GRADED`**, whose `category` is a **fourth** bucket, `"ungraded"`. Ungraded rows leave BOTH sides of every rate: `RunSummary.pass_rate` / `error_share` and `VariantAggregate.pass_rate` divide by `tasks_graded` (`tasks_run - tasks_not_graded`), and `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter. **Only SUCCESS/FAILURE collapse into it** — `ERROR`, `TIMEOUT`, `BUILD_FAILED`, `MAX_TURNS_EXHAUSTED` and the budget stops are facts about the *run*, not about grading, and still apply (so `execute` still exits non-zero on a crash). The switch is `BatchRunConfig.grade` → `Orchestrator(grade=...)` → the **four** grading call sites (single-shot, evaluate-only, the simulation dialog check, and post-failure diagnostics); it crosses the docker boundary in `context.json` (defaulting to `True` in-container, so a host predating `execute` keeps grading). It is **deliberately not a task-config field** — no 5-layer merge, no `-D` path — because a task YAML must never declare itself ungraded; only the invoking command decides. `run` and `execute` share one body (`run_command.run_pipeline`) and differ solely in that flag, so there is no third code path. Two things are refused rather than degraded: `--junit-xml` (a report of verdicts, and there are none — though `reports_junit` still emits `` for an ungraded row it encounters) and simulation tasks (their turn-continuation logic reads criteria results, so an ungraded dialog would silently change its own stopping behavior). `stop_early:` blocks are inert under `execute` for the same reason the kill switch exists: the full trajectory is the deliverable. Motivating consumer: an external harness (Harbor / Terminal-Bench 2.0) that builds its own container, calls coder-eval as the agent, and grades with its own tests. - **Detached grading (`evaluate` over a run dir) + `Sandbox.adopt`**: `coder-eval evaluate` takes two shapes, told apart by a **pure** resolver (`cli/evaluate_target.py`) on one probe — a target holding `task.json` is a run directory. Run-dir mode rebuilds the task from the run's own `task_config.resolved`, **not** by re-loading the YAML: `resolved` is post-merge, so variant overrides / `-D` / dataset expansion are already baked in and re-loading the source would silently grade a *different* task (fallback to `source_file` only when `resolved` no longer validates, and loudly). It seeds the fresh result from the prior one via `Orchestrator(prior_result=...)` → `_seed_from_prior_result`, which carries the trajectory (every derived figure — tokens, cost, `command_stats`, `model_used` — recomputes from `iterations`), `iteration_count`, execution facts, and **`early_stop` — load-bearing, because gate selection is FIRED-ONLY**: dropping it re-grades a truncated trajectory under the full-run strict-AND gate and can flip the verdict. Grader-host `environment_info` is preserved under a `graded_by` sub-dict rather than overwriting the run's. Two other parity fixes: `command_base_path` is now persisted into `environment_info` by `_sync_sandbox_command_path_with_agent` and restored in the evaluate-only branch (closing the PATH gap that method's docstring already named), and `_join_litellm_actual_cost` **skips** when `prior_result` is set (its join keys on a per-Orchestrator nonce the prior turns never carried, so it would clobber already-correct costs). The verdict is written back into the run's `task.json`, with the pre-grade record kept as `task.execute.json` — that in-place write is what makes plain `coder-eval aggregate ` rebuild a graded `run.json` with **zero** new code. **`Sandbox.adopt(workspace)`** is the grade-in-place primitive: it reuses `setup`'s adoption half but skips every *materializing* step (`_setup_template`, `_generate_cli_recorders`, venv/package installs, the destructive `$HOME` remediation), running only non-mutating derivation (mock-dir `+x`, venv *discovery*, plugin-tools pin); `_cleanup_on_exit` stays False so an adopted tree is never moved or deleted. In-place is **more correct**, not merely faster: `_setup_template` filters the copy through `_should_ignore_template_file`, which drops `node_modules` / `dist` / `build` / `.venv` / `.git`, so on the copy path a criterion like `test -f dist/bundle.js` fails as a *copying artifact* rather than as a verdict (verified: 0.00 "does not exist" on copy vs 1.00 in place). Defaults: in-place for a run dir, copy for a bare work dir (criteria can mutate it and it is the user's own tree); `--in-place`/`--copy` override. `adopt` hard-errors on `driver: docker` (a container workspace is unreachable from the host) and a re-grade refuses on a `reference_digest` mismatch. NOTE the Typer command is a thin wrapper over `run_evaluation(...)`, which has real Python defaults — calling a Typer command function in-process hands unspecified options an `OptionInfo` sentinel, which silently made `in_place=None` truthy. +- **`--resume` is command-relative**: `partition_for_resume(tasks, *, grade)` returns a four-way `ResumePartition` (`to_run` / `to_grade` / `prior_results` / `prior_resolved`), because **"finished" is not absolute — it depends on what the resuming command still owes the task**. A `NOT_GRADED` row carries a final status, so the original "has any final status" test called it complete: right for `execute --resume` (it finished executing), and wrong for `run --resume`, which was asked to grade and would instead report "already complete", grade nothing, and **exit 0**. Under `grade=True` those rows route to `to_grade`, where `_grade_resumed_tasks` runs the criteria against the trajectory and workspace already on disk via `orchestration/regrade.py::regrade_in_place` — reusing the agent spend, which is the entire reason `execute` and `run` are separate. The carve-out is **only** for `NOT_GRADED`: `FAILURE`/`ERROR` stay complete under both commands (resume has never retried failures — delete the task.json), and `clear_rerun_artifacts` deliberately skips `to_grade`, whose artifacts are the very thing being graded. A per-task grading failure is warned and folded back in with its ORIGINAL ungraded result, so one bad row neither aborts the resume nor vanishes from run.json. `grade` is in `_FINGERPRINT_DIFF_EXEMPT` because `execute` → `run --resume` is a supported flow, not config drift — and the warning's "already-finalized tasks keep their original-config results" text is actively wrong for it. **`orchestration/regrade.py` is the single implementation** shared by that path and `evaluate`'s run-dir mode (two copies would drift into two verdicts for the same run); it raises plain `RegradeError`, which the CLI wraps, since `orchestration/` must not import the CLI layer (CE004). One fidelity rule it enforces: a re-graded row keeps the **agent run's** `started_at`/`duration_seconds`, not the grading pass's — a 10-minute run re-graded in 2s would otherwise report 2s into `average_duration`, the report tables and the evalboard; the grading cost is preserved separately as `environment_info["grading_duration_seconds"]`. - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `stop_early.decide_within`, deliberately lives on `LiveSuccessCriterion` instead (see below) — the watcher must attribute a decision-step timeout to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express. - **Early stop on criterion (opt-in, per-criterion arming)**: a `stop_early:` block (`StopEarlyPolicy`) on a criterion ends a single-shot run early once the run's **armed** criteria decide the outcome, so a raised `max_turns` isn't wasted on the smoke flavor. The block's PRESENCE is the arming and alone activates the watcher — there is **no run-level master switch**: `run_limits.stop_early: false` is the run-level KILL SWITCH that force-disarms every block (the one-line experiment-variant/`-D` override for an authoritative full run), and `run_limits.stop_early: true` (the removed master arm) is a hard `EarlyStopConfigError` at resolution. The block exists on `LiveSuccessCriterion` only (currently `skill_triggered`, `command_executed` — so arming an unobservable criterion is unrepresentable, a pydantic extra-forbid error). Arming carries one implicit trigger (a native live-fail may fail-stop the run); its keys refine it: `on_pass: stop` (pass-stop the moment the criterion live-passes; default `continue` just latches) and `decide_within: N` (still undecided after N tool-call steps latches an **effective fail**, fed through the same fail-stop rule, reported as `decision_budget_exceeded` — an ordinary weighted fail, NOT a gate-bypassing force-fail; cumulative across retry attempts of the same turn). A trigger whose polarity the instance can't decide (per the abstract, checker-independent `live_decidable_polarities()`, a pure function of the criterion's own fields, paired with the checker's `live_verdict` override by lint rule CE025, a registry-based whole-tree check) is **inert by design** — one dataset-fanned YAML line serves both positive rows (pass/timeout live) and distractor rows (fail live). Verdicts **latch**: once a criterion decides, its `live_verdict` is never polled again. Stop rule is weighted, not strict-boolean: `run_limits.stop_early_gate_threshold` (default `1.0`, reproducing strict-AND behavior exactly) is the minimum weighted score (`Σ weight·score / Σ weight` over the armed subset) required to pass; a fail-stop fires once the armed set's **ceiling** (best case for everything still undecided) can no longer reach the threshold — so a low-weight fail or timeout that can't doom the gate is absorbed and the run continues — and is **deferred while any pass-capable armed criterion is undecided** (a distractor misfire never truncates a positive row's recall signal); a pass-stop fires once the `on_pass: stop` subset's **floor** (worst case) already meets the threshold, and is symmetrically **deferred while any pass-capable armed criterion outside the `on_pass: stop` subset is undecided** (so an early pass never freezes a sibling `on_pass: continue` criterion's signal out of the trajectory). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a kill-switched (`stop_early: false`) run. Driven by `orchestration/early_stop.py::EarlyStopWatcher` (built when `early_stop_active(task)`: ≥1 armed criterion, kill switch not thrown) through the agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. Gating is **FIRED-ONLY**: a run the watcher actually cut gates on the **armed subset** via the weighted `EvaluationResult.armed_criteria_passed`; a run that completes naturally — armed or not — gates strict-AND via `all_criteria_passed`, so adding a block never changes the verdict of a run it didn't cut. Note the gate keys on the watcher having FIRED (`result.early_stop is not None`), not on confirmed truncation — an agent that ignores `should_stop`, or a stop firing on the final message, still gates armed-only. Every resolution-time guardrail violation is a hard error at resolution (plan *and* run); the one load-time case — a `stop_early:` block on a non-live criterion — is a pydantic schema error at task load, which the run surface reports as a skipped task like any other malformed task. A runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo` (incl. `gate_threshold` at stop time), report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. No blocks anywhere ⇒ behavior byte-for-byte unchanged. diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 8c4454f3..c37fa46f 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -41,7 +41,7 @@ coder-eval run tasks/hello_date.yaml --stream full # live LLM output | `--driver` | Shorthand alias for `-D sandbox.driver=…` (`tempdir` or `docker`) | | `--type, -T` | Override agent type for all tasks (`claude-code`, `codex`, `antigravity`, `opencode`, or a plugin kind). | | `--repeats` | Run each `(task, variant)` N times (≥1); overrides experiment/variant `repeats:`. See [Replicates](#replicates). | -| `--resume` | Resume an interrupted run: skip tasks already finalized in `--run-dir` and run the rest, folding prior results into `run.json`. Requires `--run-dir`. A task with *any* final status (incl. FAILED/ERROR) counts as finalized, so resume does **not** retry failures — delete a task's `task.json` to force a re-run. A config mismatch is warned, not refused. | +| `--resume` | Resume an interrupted run: skip tasks already finalized in `--run-dir` and run the rest, folding prior results into `run.json`. Requires `--run-dir`. See [Resuming a run](#resuming-a-run). | | `--sample N` | For dataset-backed tasks, run a fixed-seed random N-row sample (reproducible; cheap smoke test). See [Bring Your Own Dataset](DATASETS.md). | | `--sample-per-stratum N` | For dataset-backed tasks, keep up to N rows per stratum (`stratify_field`). Overridden by `--sample`. Nondeterministic unless `dataset.sample_seed` is set — see [Bring Your Own Dataset](DATASETS.md). | | `--include-skipped` | Also run tasks marked `skip: true` in their YAML (off by default so CI keeps excluding them). | @@ -86,12 +86,47 @@ degraded: | Not supported | Why | | --- | --- | | `--junit-xml` | A JUnit report reports verdicts, and there are none. | -| `--resume` | Resume treats "has any final status" as finalized, so a `NOT_GRADED` row would be *skipped* by a later `run --resume` rather than graded. | | Simulation tasks | The dialog loop reads criteria results to decide whether to keep talking, so an ungraded dialog would silently change its own stopping behavior. Rejected by name at startup. | `stop_early:` blocks are also inert here: early stop exists to cut a run once the criteria decide the outcome, and under `execute` the full trajectory is the deliverable. +`--resume` **is** supported, and `run --resume` pairs with it (see below). + +### Resuming a run + +`--resume` continues an interrupted run without re-paying for finished work. It +requires `--run-dir` (an auto-generated directory is always fresh). + +What it owes each task depends on what it finds in that task's `task.json`: + +| On disk | `run --resume` | `execute --resume` | +| --- | --- | --- | +| No `task.json`, unreadable, or no `final_status` | re-run | re-run | +| `NOT_GRADED` | **grade in place** | already complete | +| Any other status, **including `FAILURE` / `ERROR`** | already complete | already complete | + +**"Finished" is relative to the resuming command.** A `NOT_GRADED` row owes +`execute` nothing — it finished executing — but owes `run` a grade. So +`run --resume` runs the criteria against the trajectory and workspace already on +disk instead of re-running the agent, which is the whole reason to split the two +commands: + +```bash +coder-eval execute tasks/*.yaml --run-dir ./r # expensive half +coder-eval run tasks/*.yaml --run-dir ./r --resume # grades what execute left +``` + +**Resume never retries failures.** `FAILURE` and `ERROR` count as complete under +both commands — delete a task's `task.json` to force a re-run. A task about to +re-run has its stale `artifacts/` cleared first, so leftover files from +a killed container cannot satisfy a file-based criterion. + +A run-config mismatch is **warned, not refused**: resumed tasks keep their +original-config results, so the run genuinely mixes configs. The `grade` flag is +exempt from that warning, because `execute` → `run --resume` is a supported flow +rather than a config mistake. + ### `coder-eval plan` — validate tasks ```bash diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index cdba3fc5..7de52fc3 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -19,113 +19,26 @@ TemplateDirSource, parse_agent_config, ) +from ..orchestration.regrade import ( + PRE_GRADE_JSON, + TASK_JSON, + RegradeError, + back_up_pre_grade_record, + default_workspace, + load_prior_result, + task_from_prior, + verify_reference_unchanged, +) from ..orchestration.task_loader import load_task from ..orchestrator import Orchestrator from ..sandbox import Sandbox from .console import console -from .evaluate_target import TASK_JSON, EvaluateMode, EvaluateTarget, EvaluateTargetError, resolve_evaluate_target +from .evaluate_target import EvaluateMode, EvaluateTarget, EvaluateTargetError, resolve_evaluate_target from .run_helpers import prepare_run_directory logger = logging.getLogger(__name__) -# Where a run directory keeps the workspace the agent worked in. The extra -# segment is the task id: preservation writes `artifacts//...`. -ARTIFACTS_DIRNAME = "artifacts" - - -def _load_prior_result(run_dir: Path) -> EvaluationResult: - """Read a finished run's ``task.json``.""" - raw = (run_dir / TASK_JSON).read_text(encoding="utf-8") - try: - return EvaluationResult.model_validate_json(raw) - except ValueError as e: - raise typer.BadParameter(f"{run_dir / TASK_JSON} is not a readable EvaluationResult: {e}") from e - - -def _task_from_prior(prior: EvaluationResult, run_dir: Path) -> tuple[TaskDefinition, str]: - """Rebuild the executed task from the run's own recorded config. - - Rebuilding from ``task_config.resolved`` rather than re-reading the YAML is - what makes the grade describe the run that happened: ``resolved`` is the - post-merge definition, so variant overrides, ``-D`` flags and dataset row - expansion are all already baked in. Re-loading the source YAML would silently - grade a DIFFERENT task whenever any of those were used. - - Falls back to the source YAML only when ``resolved`` will not validate (a - schema change since the run), and says so loudly — a quiet fallback would - reintroduce exactly the drift above. - """ - record = prior.task_config - if record is None: - raise typer.BadParameter( - f"{run_dir / TASK_JSON} carries no task_config, so the executed task cannot be " - + "rebuilt. Pass the task file explicitly: coder-eval evaluate " - ) - try: - return TaskDefinition.model_validate(record.resolved), record.source_yaml - except ValueError as e: - if not record.source_file or not Path(record.source_file).is_file(): - raise typer.BadParameter( - f"The resolved task config in {run_dir / TASK_JSON} no longer validates ({e}), and " - + "its source YAML is unavailable. Pass the task file explicitly." - ) from e - console.print( - f"[yellow]⚠[/] The recorded resolved config does not validate ({e}); falling back to " - + f"{record.source_file}. Variant overrides, -D flags and dataset expansion from the " - + "original run are NOT reapplied, so this grade may not match what ran." - ) - return load_task(Path(record.source_file)) - - -def _default_workspace(run_dir: Path, prior: EvaluationResult) -> Path: - """Locate the workspace a finished run left behind. - - ``sandbox_path`` is authoritative when it still exists — it is where the run - actually worked. Otherwise fall back to the preserved artifacts tree, whose - single child is named for the task. - """ - if prior.sandbox_path: - recorded = Path(prior.sandbox_path) - if recorded.is_dir(): - return recorded - - artifacts = run_dir / ARTIFACTS_DIRNAME - if not artifacts.is_dir(): - raise typer.BadParameter( - f"No workspace to grade: {artifacts} does not exist and the recorded sandbox_path " - + f"({prior.sandbox_path or 'unset'}) is gone. The run was probably made with " - + "--preservation-mode NONE. Point at one explicitly with --workspace." - ) - children = [p for p in sorted(artifacts.iterdir()) if p.is_dir()] - # Preservation nests the workspace under the task id; a flat artifacts dir - # (no subdirectory) means the workspace IS artifacts/. - return children[0] if len(children) == 1 else artifacts - - -def _verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition) -> None: - """Refuse to grade when the reference tree changed since the run. - - ``reference_comparison`` and reference-carrying judges score against - ``task.reference.directory``. If it moved since the run, the re-grade would - silently measure the agent's old work against a new answer key. - """ - recorded = prior.environment_info.get("reference_digest") - if not isinstance(recorded, str) or task.reference is None: - return - from ..orchestration.evaluation import resolve_reference_dir - from ..path_utils import digest_tree - - resolved = resolve_reference_dir(task, None) - if resolved is None or not resolved.is_dir(): - return - if digest_tree(resolved) != recorded: - raise typer.BadParameter( - f"The reference directory {resolved} changed since this run was executed " - + "(digest mismatch). Grading now would score the agent's work against a " - + "different answer key. Restore the reference, or re-run the task." - ) - @dataclass(frozen=True) class _ResolvedInputs: @@ -156,15 +69,25 @@ def _resolve_inputs(task_or_run_dir: Path, work_dir: Path | None, workspace: Pat + "directory to grade is already the second argument." ) + try: + return _resolve_run_dir_or_work_dir(target, workspace) + except RegradeError as e: + # The shared core raises a plain exception (orchestration/ must not + # depend on the CLI layer, CE004); surface it as a CLI error here. + raise typer.BadParameter(str(e)) from e + + +def _resolve_run_dir_or_work_dir(target: EvaluateTarget, workspace: Path | None) -> _ResolvedInputs: + """The mode-specific half of :func:`_resolve_inputs`.""" prior: EvaluationResult | None = None if target.mode is EvaluateMode.RUN_DIR: - prior = _load_prior_result(target.target) + prior = load_prior_result(target.target) if target.task_file is not None: task, source_yaml = load_task(target.task_file) console.print(f"[dim]Grading with {target.task_file} (overrides the run's recorded config).[/dim]") else: - task, source_yaml = _task_from_prior(prior, target.target) - work_dir = workspace or _default_workspace(target.target, prior) + task, source_yaml = task_from_prior(prior, target.target) + work_dir = workspace or default_workspace(target.target, prior) recorded_source = prior.task_config.source_file if prior.task_config else None task_file = target.task_file or (Path(recorded_source) if recorded_source else None) else: @@ -192,7 +115,7 @@ def _resolve_inputs(task_or_run_dir: Path, work_dir: Path | None, workspace: Pat task.agent = parse_agent_config(**{**task.agent.model_dump(exclude_unset=True), "type": AgentKind.CLAUDE_CODE}) if prior is not None: - _verify_reference_unchanged(prior, task) + verify_reference_unchanged(prior, task) return _ResolvedInputs( target=target, @@ -434,10 +357,9 @@ def _write_back(run_dir: Path, result: EvaluationResult) -> None: only evidence that the run was executed separately. """ target = run_dir / TASK_JSON - backup = run_dir / "task.execute.json" + backup = run_dir / PRE_GRADE_JSON + back_up_pre_grade_record(run_dir) try: - if not backup.exists(): - backup.write_text(target.read_text(encoding="utf-8"), encoding="utf-8") target.write_text(result.model_dump_json(indent=2), encoding="utf-8") except OSError as e: # Never fail the grade over the write-back: the verdict was computed and diff --git a/src/coder_eval/cli/execute_command.py b/src/coder_eval/cli/execute_command.py index b90fa0e3..79388003 100644 --- a/src/coder_eval/cli/execute_command.py +++ b/src/coder_eval/cli/execute_command.py @@ -12,14 +12,13 @@ be worse than not grading at all: coder-eval's verdict would be reported alongside Harbor's without being the one that counts. -Every flag on ``run`` is available here except two, and both omissions are -deliberate: +Every flag on ``run`` is available here except ``--junit-xml``, which is a report +of verdicts and there are none. -* ``--junit-xml`` — a JUnit report is a report of verdicts, and there are none. -* ``--resume`` — ``partition_for_resume`` treats "has any final status" as - finalized, so a ``NOT_GRADED`` row would be skipped by a later ``run --resume`` - rather than graded. Supporting it needs resume to distinguish "done" from - "executed but unscored"; until then, refusing is the honest option. +``--resume`` IS supported, because ``partition_for_resume`` now takes the +resuming command into account: a ``NOT_GRADED`` row owes ``execute`` nothing (it +finished executing) but owes ``run`` a grade, so ``run --resume`` grades those +rows in place rather than skipping them as "already complete". The command shares ``run``'s entire body (``run_command.run_pipeline``); only the Typer signature is restated, because Typer builds its parser from the signature. @@ -55,6 +54,16 @@ def execute_command( "--run-dir", help="Custom run directory (default: auto-generated timestamped directory in runs/)", ), + resume: bool = typer.Option( + False, + "--resume", + help=( + "Resume an interrupted execute: skip tasks already executed in --run-dir " + "and run only the rest. Requires --run-dir. A NOT_GRADED row counts as " + "done here (it finished executing); a later `coder-eval run --resume` on " + "the same directory grades those rows instead of skipping them." + ), + ), max_parallel: int = typer.Option( 1, "--max-parallel", @@ -187,8 +196,7 @@ def execute_command( ERROR / TIMEOUT / TOKEN_BUDGET_EXCEEDED and exits non-zero exactly as under `run`. Only the verdict is withheld, never the facts of the run. - Not supported here: --junit-xml (no verdicts to report), --resume (a - NOT_GRADED row would be mistaken for a finalized one), and simulation tasks + Not supported here: --junit-xml (no verdicts to report) and simulation tasks (their turn-continuation logic reads criteria results). Examples: @@ -202,8 +210,8 @@ def execute_command( task_files=task_files, preservation_mode=preservation_mode, run_dir=run_dir, - # Not exposed as flags — see the module docstring for why each is refused. - resume=False, + resume=resume, + # Not exposed as a flag — a JUnit report reports verdicts, and there are none. junit_xml=None, max_parallel=max_parallel, verbose=verbose, diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index d820e301..65337c65 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -657,6 +657,70 @@ def _on_task_complete(result: Any) -> None: return result +async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[ResolvedTask, TaskResult]]: + """Grade the rows ``coder-eval execute`` left NOT_GRADED, in place. + + Each task's trajectory and workspace are already on disk, so this runs the + criteria against them instead of re-running the agent — that reuse is the + whole reason to split ``execute`` from ``run``. + + The task config comes from ``rt.task`` (this run's own 5-layer resolution), + not from the recorded one: ``--resume`` re-resolves the same task files, and + a config that drifted since the execute is already surfaced by the run + fingerprint warning above. + + A task that cannot be graded is reported and folded back in with its ORIGINAL + ungraded result, so one bad row neither aborts the resume nor silently + vanishes from run.json — it stays visible as ``tasks_not_graded``. + """ + from ..orchestration.regrade import ( + RegradeError, + back_up_pre_grade_record, + default_workspace, + load_prior_result, + regrade_in_place, + verify_reference_unchanged, + ) + + graded: list[tuple[ResolvedTask, TaskResult]] = [] + for rt in to_grade: + prior = load_prior_result(rt.run_dir) + try: + verify_reference_unchanged(prior, rt.task) + workspace = default_workspace(rt.run_dir, prior) + # Preserve the ungraded record BEFORE the orchestrator overwrites + # task.json in this same directory. + back_up_pre_grade_record(rt.run_dir) + result = await regrade_in_place( + task=rt.task, + prior=prior, + workspace=workspace, + run_dir=rt.run_dir, + task_file=rt.task_file, + source_yaml=rt.source_yaml, + variant_id=rt.variant_id, + replicate_index=rt.replicate_index, + ) + except (RegradeError, OSError, RuntimeError, ValueError) as e: + console.print(f"[yellow]⚠[/] Could not grade {rt.task.task_id}: {e}") + result = prior + graded.append( + ( + rt, + TaskResult( + task_id=rt.task.task_id, + variant_id=rt.variant_id, + result=result, + duration=result.duration_seconds or 0.0, + suite_id=rt.task.suite_id, + row_id=rt.task.row_id, + replicate_index=rt.replicate_index, + ), + ) + ) + return graded + + async def _run_with_experiment( all_task_files: list[Path], config: BatchRunConfig, @@ -787,16 +851,26 @@ async def _run_with_experiment( prior_results: list[TaskResult] = [] prior_resolved: list[ResolvedTask] = [] if resume: - to_run, prior_results, prior_resolved = partition_for_resume(resolved) + part = partition_for_resume(resolved, grade=grade) + to_run, prior_results, prior_resolved = part.to_run, part.prior_results, part.prior_resolved # A re-run task re-executes from scratch, so any leftover artifacts (only # DIRECT_WRITE writes them live; a container killed mid-run leaves partials) # are stale and could let a file-based criterion pass on the old output. + # to_grade is deliberately NOT cleared: its artifacts are the run's output + # and the very thing being graded. cleared = clear_rerun_artifacts(to_run) console.print( f"[cyan]↻ Resume:[/] {len(prior_results)} task(s) already complete, " + f"running {len(to_run)} remaining" + + (f", grading {len(part.to_grade)} executed-but-ungraded" if part.to_grade else "") + (f" (cleared {cleared} stale artifact dir(s))" if cleared else "") ) + # Grade the rows `execute` left behind, reusing the trajectory and + # workspace already on disk rather than paying for the agent twice. + # Folded in as prior_results so the summary covers them like any other. + for rt, tr in await _grade_resumed_tasks(part.to_grade): + prior_results.append(tr) + prior_resolved.append(rt) # Print execution mode print_execution_mode(len(to_run), max_parallel) diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py index d79e6d96..d9fe0098 100644 --- a/src/coder_eval/orchestration/batch.py +++ b/src/coder_eval/orchestration/batch.py @@ -15,7 +15,7 @@ from collections.abc import Callable, Iterator from datetime import datetime from pathlib import Path -from typing import Any +from typing import Any, NamedTuple from ..models import ( AgentKind, @@ -324,10 +324,24 @@ def _create_error_task_result( ) -def partition_for_resume( - resolved_tasks: list[ResolvedTask], -) -> tuple[list[ResolvedTask], list[TaskResult], list[ResolvedTask]]: - """Split resolved tasks into (to_run, prior_results, prior_resolved) for --resume. +class ResumePartition(NamedTuple): + """How ``--resume`` splits a run's tasks over what each one still needs.""" + + to_run: list[ResolvedTask] + """Never finished executing — re-run from scratch.""" + + to_grade: list[ResolvedTask] + """Executed but ungraded (NOT_GRADED). Needs criteria, NOT another agent run.""" + + prior_results: list[TaskResult] + """Genuinely finished — reloaded so run.json covers the whole suite.""" + + prior_resolved: list[ResolvedTask] + """The ResolvedTask for each entry of prior_results, same order.""" + + +def partition_for_resume(resolved_tasks: list[ResolvedTask], *, grade: bool = True) -> ResumePartition: + """Split resolved tasks over what ``--resume`` still owes each one. A task is already-complete when its task.json exists, parses, and carries a final_status. task.json is written atomically at end-of-run, so any parseable @@ -335,26 +349,40 @@ def partition_for_resume( tasks are reloaded into TaskResults (to fold into run.json) and excluded from to_run; everything else — including failed-to-parse — re-runs. + **"Finished" is relative to the resuming command, not absolute.** A + ``NOT_GRADED`` row (written by ``coder-eval execute``) has a final status, so + the naive test calls it complete. That is right for ``execute --resume``, + which owes it nothing — and wrong for ``run --resume``, which was asked to + grade: skipping it would report "already complete", grade nothing, and exit + 0. So under ``grade=True`` those rows go to ``to_grade`` instead, where the + caller runs the criteria against the trajectory and workspace already on + disk rather than paying for the agent a second time. + + Note the asymmetry is only for NOT_GRADED. FAILURE and ERROR stay complete + under both commands — resume has never retried failures (delete a task's + task.json to force that), and this does not change it. + Args: resolved_tasks: Fully-resolved tasks for the whole run. + grade: Whether the resuming command grades (``run``) or not (``execute``). Returns: - (to_run, prior_results, prior_resolved): - - to_run: tasks still needing execution - - prior_results: reloaded results for already-complete tasks - - prior_resolved: the ResolvedTask for each prior_result (same order) + The four-way :class:`ResumePartition`. """ to_run: list[ResolvedTask] = [] + to_grade: list[ResolvedTask] = [] prior_results: list[TaskResult] = [] prior_resolved: list[ResolvedTask] = [] for rt in resolved_tasks: tr = _load_completed_result(rt) if tr is None: to_run.append(rt) + elif grade and tr.result.final_status.category == "ungraded": + to_grade.append(rt) else: prior_results.append(tr) prior_resolved.append(rt) - return to_run, prior_results, prior_resolved + return ResumePartition(to_run, to_grade, prior_results, prior_resolved) def clear_rerun_artifacts(to_run: list[ResolvedTask]) -> int: @@ -511,13 +539,24 @@ def read_run_fingerprint(run_dir: Path) -> dict[str, object] | None: return data if isinstance(data, dict) else None +# `grade` is excluded from the drift warning: `execute` then `run --resume` is a +# SUPPORTED flow, not a config mistake, and the warning's text ("already-finalized +# tasks keep their original-config results") is actively wrong for it — those rows +# are re-graded with the current config, which is the entire point. +_FINGERPRINT_DIFF_EXEMPT = frozenset({"grade"}) + + def fingerprint_diff(prior: dict[str, object], current: dict[str, object]) -> dict[str, tuple[object, object]]: """Keys present in BOTH stamps that disagree, as ``{key: (prior, current)}``. Only keys present in ``prior`` are compared, so adding fingerprint fields in a later version never false-flags a resume of an older run. """ - return {k: (prior[k], current[k]) for k in current if k in prior and prior[k] != current[k]} + return { + k: (prior[k], current[k]) + for k in current + if k in prior and prior[k] != current[k] and k not in _FINGERPRINT_DIFF_EXEMPT + } def _override_uip_versions_from_tasks(version_info: dict[str, Any], task_results: list[TaskResult]) -> None: diff --git a/src/coder_eval/orchestration/regrade.py b/src/coder_eval/orchestration/regrade.py new file mode 100644 index 00000000..aa6b15c7 --- /dev/null +++ b/src/coder_eval/orchestration/regrade.py @@ -0,0 +1,205 @@ +"""Grade a run that already executed — the shared core behind two callers. + +``coder-eval execute`` leaves every row ``NOT_GRADED``. Two commands can supply +the verdict afterwards, and both must do it identically: + +* ``coder-eval evaluate `` — grade one finished task explicitly. +* ``coder-eval run --resume`` — grade the ungraded rows it finds in the run dir + instead of re-executing them (see ``partition_for_resume``). + +The logic lives here rather than in ``cli/`` because the resume path is not a CLI +concern, and because two copies of "how to re-grade" would drift into two +different verdicts for the same run. Errors surface as :class:`RegradeError`, a +plain exception the CLI wraps into its own error type — ``orchestration/`` must +not depend on the CLI layer (CE004). +""" + +from __future__ import annotations + +import asyncio +import logging +from pathlib import Path + +from coder_eval.models import EvaluationResult, PreservationMode, TaskDefinition +from coder_eval.sandbox import Sandbox + + +logger = logging.getLogger(__name__) + +TASK_JSON = "task.json" +PRE_GRADE_JSON = "task.execute.json" +ARTIFACTS_DIRNAME = "artifacts" + + +class RegradeError(Exception): + """A finished run cannot be re-graded as asked.""" + + +def load_prior_result(run_dir: Path) -> EvaluationResult: + """Read a finished run's ``task.json``.""" + path = run_dir / TASK_JSON + try: + raw = path.read_text(encoding="utf-8") + except OSError as e: + raise RegradeError(f"Cannot read {path}: {e}") from e + try: + return EvaluationResult.model_validate_json(raw) + except ValueError as e: + raise RegradeError(f"{path} is not a readable EvaluationResult: {e}") from e + + +def task_from_prior(prior: EvaluationResult, run_dir: Path) -> tuple[TaskDefinition, str]: + """Rebuild the executed task from the run's own recorded config. + + Rebuilding from ``task_config.resolved`` rather than re-reading the YAML is + what makes the grade describe the run that happened: ``resolved`` is the + post-merge definition, so variant overrides, ``-D`` flags and dataset row + expansion are all already baked in. Re-loading the source YAML would silently + grade a DIFFERENT task whenever any of those were used. + + Falls back to the source YAML only when ``resolved`` will not validate (a + schema change since the run), and says so loudly — a quiet fallback would + reintroduce exactly the drift above. + """ + from .task_loader import load_task + + record = prior.task_config + if record is None: + raise RegradeError( + f"{run_dir / TASK_JSON} carries no task_config, so the executed task cannot be " + + "rebuilt. Pass the task file explicitly: coder-eval evaluate " + ) + try: + return TaskDefinition.model_validate(record.resolved), record.source_yaml + except ValueError as e: + if not record.source_file or not Path(record.source_file).is_file(): + raise RegradeError( + f"The resolved task config in {run_dir / TASK_JSON} no longer validates ({e}), and " + + "its source YAML is unavailable. Pass the task file explicitly." + ) from e + logger.warning( + "The recorded resolved config does not validate (%s); falling back to %s. Variant " + + "overrides, -D flags and dataset expansion from the original run are NOT reapplied, " + + "so this grade may not match what ran.", + e, + record.source_file, + ) + return load_task(Path(record.source_file)) + + +def default_workspace(run_dir: Path, prior: EvaluationResult) -> Path: + """Locate the workspace a finished run left behind. + + ``sandbox_path`` is authoritative when it still exists — it is where the run + actually worked. Otherwise fall back to the preserved artifacts tree, whose + single child is named for the task. + """ + if prior.sandbox_path: + recorded = Path(prior.sandbox_path) + if recorded.is_dir(): + return recorded + + artifacts = run_dir / ARTIFACTS_DIRNAME + if not artifacts.is_dir(): + raise RegradeError( + f"No workspace to grade: {artifacts} does not exist and the recorded sandbox_path " + + f"({prior.sandbox_path or 'unset'}) is gone. The run was probably made with " + + "--preservation-mode NONE." + ) + children = [p for p in sorted(artifacts.iterdir()) if p.is_dir()] + # Preservation nests the workspace under the task id; a flat artifacts dir + # (no subdirectory) means the workspace IS artifacts/. + return children[0] if len(children) == 1 else artifacts + + +def verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition) -> None: + """Refuse to grade when the reference tree changed since the run. + + ``reference_comparison`` and reference-carrying judges score against + ``task.reference.directory``. If it moved since the run, the re-grade would + silently measure the agent's old work against a new answer key. + """ + recorded = prior.environment_info.get("reference_digest") + if not isinstance(recorded, str) or task.reference is None: + return + from coder_eval.path_utils import digest_tree + + from .evaluation import resolve_reference_dir + + resolved = resolve_reference_dir(task, None) + if resolved is None or not resolved.is_dir(): + return + if digest_tree(resolved) != recorded: + raise RegradeError( + f"The reference directory {resolved} changed since this run was executed " + + "(digest mismatch). Grading now would score the agent's work against a " + + "different answer key. Restore the reference, or re-run the task." + ) + + +def back_up_pre_grade_record(run_dir: Path) -> None: + """Keep the ungraded ``task.json`` beside the graded one, once. + + The write-back replaces the only on-disk evidence that this run was executed + separately from grading. Copying it first keeps that auditable. Written once: + a second grade must not overwrite the ORIGINAL execute record with an + already-graded one. + """ + source, backup = run_dir / TASK_JSON, run_dir / PRE_GRADE_JSON + if backup.exists() or not source.is_file(): + return + try: + backup.write_text(source.read_text(encoding="utf-8"), encoding="utf-8") + except OSError as e: + # Never fail a grade over the audit copy. + logger.warning("Could not preserve the pre-grade record at %s: %s", backup, e) + + +async def regrade_in_place( + *, + task: TaskDefinition, + prior: EvaluationResult, + workspace: Path, + run_dir: Path, + task_file: Path | None, + source_yaml: str, + variant_id: str, + replicate_index: int = 0, +) -> EvaluationResult: + """Run ``task``'s criteria against an already-executed ``workspace``. + + The workspace is *adopted*, never copied: it is the run's own output, and the + template-copy path filters out ``node_modules`` / ``dist`` / ``build`` / + ``.venv``, which would make a criterion reading those fail as a copying + artifact rather than as a verdict. + + ``prior`` supplies the trajectory and the run's execution facts (see + ``Orchestrator._seed_from_prior_result``), so criteria that read the agent's + tool calls score exactly as they would have during the run. + """ + from coder_eval.orchestrator import Orchestrator + + # Grading never runs a container: the docker driver dispatches through + # DockerRunner, which needs an agent. Force tempdir so a task whose YAML says + # `driver: docker` is still gradeable on the host. + sandbox_config = task.sandbox.model_copy(deep=True).model_copy(update={"driver": "tempdir"}) + sandbox = Sandbox( + sandbox_config, + task_id=task.task_id, + task_dir=task_file.parent.resolve() if task_file is not None else None, + ) + await asyncio.to_thread(sandbox.adopt, workspace) + + orchestrator = Orchestrator( + task=task, + run_dir=run_dir, + # The workspace belongs to the run being graded; never move or delete it. + preservation_mode=PreservationMode.NONE, + task_file=task_file, + sandbox=sandbox, + variant_id=variant_id, + source_yaml=source_yaml, + replicate_index=replicate_index, + prior_result=prior, + ) + return await orchestrator.run() diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 0549e7b7..e9c991ea 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -735,13 +735,20 @@ def _seed_from_prior_result(self) -> None: silently disagrees with the run it grades. Deliberately NOT carried: ``final_status``, ``weighted_score`` and - ``success_criteria_results`` — those are exactly what this pass recomputes - — and the timestamps/duration, which describe the grading pass. + ``success_criteria_results`` — those are exactly what this pass recomputes. """ prior = self.prior_result if prior is None or self.result is None: return + # A task row describes the TASK, so its clock is the agent run's, not the + # grading pass's. Left alone, a 10-minute run re-graded in 2 seconds would + # report 2 seconds — and that figure feeds average_duration, the report + # tables and the evalboard, so harness-vs-harness comparisons would be + # quietly wrong. _finalize_result restores the duration after its own + # timing write; the grading pass's cost is recorded separately there. + self.result.started_at = prior.started_at + # The trajectory itself. Every derived figure in _finalize_result — # token totals, cost, command_stats, model_used, assistant turns — # recomputes from `iterations`, so seeding it reproduces them exactly. @@ -952,6 +959,13 @@ def _finalize_result(self, start_time: float) -> None: self.result.completed_at = datetime.now() self.result.duration_seconds = time.time() - start_time + # Re-grade: the row keeps the agent run's duration (see + # _seed_from_prior_result). The grading pass's own cost is preserved + # alongside rather than discarded, so a slow judge is still visible. + if self.prior_result is not None: + self.result.environment_info["grading_duration_seconds"] = round(self.result.duration_seconds, 3) + self.result.duration_seconds = self.prior_result.duration_seconds + # Weighted score. This call site is wrapped because _finalize_result runs # inside run()'s finally — an unguarded raise here would skip persistence and # lose task.json. The other calculate_weighted_score calls (the simulation diff --git a/tests/test_execute_command.py b/tests/test_execute_command.py index 5f0f02da..be27c2a8 100644 --- a/tests/test_execute_command.py +++ b/tests/test_execute_command.py @@ -209,7 +209,6 @@ def _option_names(command: str) -> set[str]: # risk this test exists to close: a flag added to `run` must be added here too, # or consciously listed below as a deliberate omission. _DELIBERATELY_ABSENT_FROM_EXECUTE = { - "--resume", # partition_for_resume would treat a NOT_GRADED row as finalized "--junit-xml", # a report of verdicts, and there are none } diff --git a/tests/test_execute_evaluate_loop.py b/tests/test_execute_evaluate_loop.py index c9c7264b..9ab17528 100644 --- a/tests/test_execute_evaluate_loop.py +++ b/tests/test_execute_evaluate_loop.py @@ -145,3 +145,71 @@ def test_evaluate_rejects_workspace_flag_outside_run_dir_mode(tmp_path: Path) -> result = runner.invoke(app, ["evaluate", str(AGENTLESS_TASK), str(work), "--workspace", str(work)]) assert result.exit_code != 0 assert "run directory only" in result.output + + +# -------------------------------------------------------------------------- +# `run --resume` over an executed run dir +# -------------------------------------------------------------------------- + + +def test_run_resume_grades_the_ungraded_rows_it_finds(tmp_path: Path) -> None: + """The whole point of the resume fix: `run --resume` over an executed run + must GRADE those rows, not report "already complete" and exit 0.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.NOT_GRADED.value + + result = _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert "grading 1" in result.output + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.SUCCESS.value + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert summary["tasks_not_graded"] == 0 + assert summary["tasks_succeeded"] == 1 + assert summary["pass_rate"] == 1.0 + + +def test_run_resume_does_not_re_execute_the_agent(tmp_path: Path) -> None: + """Grading must reuse the trajectory on disk. Re-executing would discard the + expensive half — the reason `execute` and `run` were split at all.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + executed = _row(_task_dir(run_dir)) + + result = _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert "running 0 remaining" in result.output, "the task was re-executed instead of graded" + graded = _row(_task_dir(run_dir)) + assert len(graded["iterations"]) == len(executed["iterations"]) + # The row still describes the TASK, not the grading pass: a re-execution + # would restamp these, and reporting the grading pass's 2s as the task's + # duration would corrupt average_duration and every harness comparison. + assert graded["started_at"] == executed["started_at"] + assert graded["duration_seconds"] == executed["duration_seconds"] + # The grading pass's own cost is kept alongside, not discarded. + assert "grading_duration_seconds" in graded["environment_info"] + # The pre-grade record is preserved by this path too, not just by `evaluate`. + assert _row(_task_dir(run_dir), "task.execute.json")["final_status"] == FinalStatus.NOT_GRADED.value + + +def test_execute_resume_treats_an_executed_row_as_done(tmp_path: Path) -> None: + """`execute --resume` owes a NOT_GRADED row nothing — it finished executing.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + + result = _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert "1 task(s) already complete" in result.output + assert "grading" not in result.output + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.NOT_GRADED.value + + +def test_execute_to_run_resume_emits_no_config_drift_warning(tmp_path: Path) -> None: + """`grade` is exempt from the fingerprint diff: this flow is supported, and + the warning's "keeps their original-config results" text is wrong for it.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + + result = _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert "run config changed" not in result.output diff --git a/tests/test_resume.py b/tests/test_resume.py index 6f46084c..21282154 100644 --- a/tests/test_resume.py +++ b/tests/test_resume.py @@ -84,7 +84,7 @@ def test_partition_splits_finalized_from_pending(tmp_path): partial.run_dir.mkdir(parents=True, exist_ok=True) (partial.run_dir / "task.json").write_text(json.dumps({"task_id": "partial_task"}), encoding="utf-8") - to_run, prior_results, prior_resolved = partition_for_resume([done, pending, partial]) + to_run, _to_grade, prior_results, prior_resolved = partition_for_resume([done, pending, partial]) assert {rt.task.task_id for rt in to_run} == {"pending_task", "partial_task"} assert [tr.task_id for tr in prior_results] == ["done_task"] @@ -94,10 +94,77 @@ def test_partition_splits_finalized_from_pending(tmp_path): assert prior_results[0].duration == 12.5 +def test_partition_sends_ungraded_rows_to_grading_not_to_rerun(tmp_path): + """`run --resume` owes a NOT_GRADED row a GRADE, not another agent run. + + A NOT_GRADED row (written by `coder-eval execute`) carries a final status, so + the naive "has any final status" test calls it complete — which made + `run --resume` report "already complete", grade nothing, and exit 0. + """ + ungraded = _resolved(tmp_path, "ungraded_task") + graded = _resolved(tmp_path, "graded_task") + _write_task_json(ungraded, FinalStatus.NOT_GRADED) + _write_task_json(graded, FinalStatus.SUCCESS) + + part = partition_for_resume([ungraded, graded], grade=True) + + assert [rt.task.task_id for rt in part.to_grade] == ["ungraded_task"] + assert part.to_run == [], "an executed row must not be re-executed — that discards the agent spend" + assert [tr.task_id for tr in part.prior_results] == ["graded_task"] + + +def test_partition_treats_ungraded_as_done_for_execute(tmp_path): + """`execute --resume` owes a NOT_GRADED row nothing: it finished executing.""" + ungraded = _resolved(tmp_path, "ungraded_task") + _write_task_json(ungraded, FinalStatus.NOT_GRADED) + + part = partition_for_resume([ungraded], grade=False) + + assert part.to_grade == [] + assert part.to_run == [] + assert [tr.task_id for tr in part.prior_results] == ["ungraded_task"] + + +@pytest.mark.parametrize("status", [FinalStatus.FAILURE, FinalStatus.ERROR, FinalStatus.TIMEOUT]) +def test_partition_still_never_retries_failures(tmp_path, status): + """The NOT_GRADED carve-out must not become a general 'retry bad rows' rule. + + Resume has never retried failures (delete a task's task.json to force that), + and both commands must keep treating them as complete. + """ + failed = _resolved(tmp_path, "failed_task") + _write_task_json(failed, status) + + for grade in (True, False): + part = partition_for_resume([failed], grade=grade) + assert part.to_run == [], f"grade={grade} re-ran a {status.value} row" + assert part.to_grade == [], f"grade={grade} tried to re-grade a {status.value} row" + assert len(part.prior_results) == 1 + + +def test_grade_flag_is_exempt_from_the_resume_drift_warning(tmp_path): + """`execute` then `run --resume` is a supported flow, not a config mistake. + + The warning's text ("already-finalized tasks keep their original-config + results") is actively wrong for it: those rows are re-graded with the current + config, which is the whole point. + """ + executed = BatchRunConfig(run_dir=tmp_path, grade=False) + write_run_fingerprint(tmp_path, compute_run_fingerprint(executed, "exp1", "direct", None)) + prior = read_run_fingerprint(tmp_path) + + grading = BatchRunConfig(run_dir=tmp_path, grade=True) + assert fingerprint_diff(prior, compute_run_fingerprint(grading, "exp1", "direct", None)) == {} + + # ...but a real difference alongside it is still reported. + both = BatchRunConfig(run_dir=tmp_path, grade=True, overrides={"agent.model": "opus"}) + assert "overrides" in fingerprint_diff(prior, compute_run_fingerprint(both, "exp1", "direct", None)) + + def test_partition_no_run_dir_yields_all_pending(tmp_path): """--resume on a fresh dir degrades to a normal run (everything to_run).""" tasks = [_resolved(tmp_path, f"t{i}") for i in range(3)] - to_run, prior_results, prior_resolved = partition_for_resume(tasks) + to_run, _to_grade, prior_results, prior_resolved = partition_for_resume(tasks) assert len(to_run) == 3 assert prior_results == [] assert prior_resolved == [] @@ -114,7 +181,7 @@ async def test_run_batch_folds_prior_into_run_json(tmp_path): _write_task_json(p_success, FinalStatus.SUCCESS) _write_task_json(p_fail, FinalStatus.FAILURE) - _, prior_results, prior_resolved = partition_for_resume([p_success, p_fail]) + _, _to_grade, prior_results, prior_resolved = partition_for_resume([p_success, p_fail]) assert len(prior_results) == 2 # Nothing left to run — exercises the merge + run.json write in isolation. From e831e92946ae80b45b17780c96e2c6b318dc0a57 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Thu, 3 Sep 2026 15:14:17 -0700 Subject: [PATCH 04/11] fix(eval): close the verdict-correctness gaps in detached grading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Full code review of the branch found two criticals and twelve highs. Every one of them is invisible to ruff/pyright/pytest/bandit/CodeQL, and every one of the worst produces a plausible number that is wrong rather than a crash. Verdict correctness * Gate selection is FIRED-ONLY, but only the AGENT path implemented it. The evaluate-only branch — the one a detached grade actually takes — called `all_criteria_passed` unconditionally, so `evaluate ` over an early-stopped run applied the full-run strict-AND gate to a truncated trajectory and could flip SUCCESS to FAILURE, then persist it. `early_stop` was seeded and read by nothing. Both paths now go through one `Orchestrator._select_gate()`. * `run()` calls the pre/post-run hooks unconditionally with `cwd = sandbox_dir`. On an adopted sandbox that is the agent's own output, and in-tree tasks stage fixtures there (`cp -a /app/[!.]* "$PWD/"`), so a detached grade overwrote the deliverables before the criteria read them. `Sandbox.was_adopted` now skips both, and their recorded results are carried from the prior run. * Grading may only move NOT_GRADED to SUCCESS/FAILURE. A prior TIMEOUT / ERROR / budget stop is an execution fact this pass neither repeated nor observed; `FinalStatus.is_execution_fact` (explicit map, no catch-all) preserves it. * The `reference_digest` guard was dead code — one grep hit in the whole tree, the read itself. The digest is now persisted at staging, resolves against the real task file, and RAISES on a vanished reference instead of returning. Counting and reporting * The evalboard rendered a clean `execute` run as 0% pass, N failed: every rate helper is `else failed++`, so an ungraded row was counted as a failure AND kept in the denominator. `StatusCategory` gains an explicit "ungraded" member; run-view, trends and watchlist exclude it from both sides. * `VariantResult.weighted_score` is `float | None`; `or 0.0` was laundering the ungraded None into a real-looking 0.000 that `_pick_best_variant` then ranked. * `SuiteRollup` gets the fourth bucket its two siblings have, plus the row-count invariant it was missing. `tasks_graded` is serialized on both aggregates. * `run --resume` exited 0 when every grade failed. The gate counts `tasks_not_graded` when grade is True; the reason is stamped on the row. Other * `evaluate`'s run-dir mode delegates to `regrade_in_place` instead of restating it. The copies had already drifted (hardcoded `replicate_index=0`). * `execute --driver docker` against an image predating `execute` silently graded; the returned row is now asserted NOT_GRADED. * A PATH restored from a run's own task.json is prepended ahead of the host's, so entries that do not exist or lie inside the graded workspace are dropped; shell commands rebuilt from a run dir's recorded config are announced. * `_seed_from_prior_result` also carries `agent_config`, `error_log_tail`, `expected_commands`, `simulation` and `sandbox_path`, which it was dropping. Tests: `test_seed_from_prior_result.py` partitions every `EvaluationResult` field as CARRIED or RECOMPUTED and fails closed on a new one; `test_regrade.py` covers the refusal branches (the digest guard's own test never reached it — the fixture had no reference, which is why the missing writer went unnoticed); `status.test.ts` covers the evalboard mirror, which had no test at all. make verify green (4654 passed, 92.14%); evalboard 621 tests green. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 4 +- evalboard/app/runs/[id]/run-view.tsx | 13 +- evalboard/lib/__tests__/status.test.ts | 70 ++++++ evalboard/lib/runs.ts | 5 + evalboard/lib/status.ts | 24 +- evalboard/lib/trends.ts | 6 +- evalboard/lib/watchlist.ts | 9 +- src/coder_eval/cli/evaluate_command.py | 31 ++- src/coder_eval/cli/run_command.py | 16 +- src/coder_eval/isolation/docker_runner.py | 21 ++ src/coder_eval/models/enums.py | 32 +++ src/coder_eval/models/experiment.py | 13 +- src/coder_eval/models/results.py | 41 +++- src/coder_eval/orchestration/experiment.py | 23 +- src/coder_eval/orchestration/regrade.py | 95 +++++-- src/coder_eval/orchestrator.py | 214 ++++++++++++---- src/coder_eval/reports.py | 12 +- src/coder_eval/reports_experiment.py | 11 +- src/coder_eval/reports_html.py | 17 +- src/coder_eval/reports_stats.py | 19 ++ src/coder_eval/sandbox.py | 7 + tests/test_cleanup_preservation_guard.py | 4 + tests/test_cli_telemetry.py | 4 +- tests/test_execute_evaluate_loop.py | 49 ++++ tests/test_post_run.py | 20 +- tests/test_pre_run.py | 32 +-- tests/test_regrade.py | 259 ++++++++++++++++++++ tests/test_run_command_junit.py | 2 +- tests/test_run_metrics.py | 4 +- tests/test_seed_from_prior_result.py | 272 +++++++++++++++++++++ 30 files changed, 1192 insertions(+), 137 deletions(-) create mode 100644 evalboard/lib/__tests__/status.test.ts create mode 100644 tests/test_regrade.py create mode 100644 tests/test_seed_from_prior_result.py diff --git a/CLAUDE.md b/CLAUDE.md index f4e4642e..2b8ac212 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -151,8 +151,8 @@ action.yml # Published composite GitHub Action (coder-ev - **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex and Antigravity (both run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced), and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. Full table + rationale: docs/agents/HARNESS_PARITY.md. - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly. - **Execute vs. run (the grading switch)**: `coder-eval execute` is `coder-eval run` with grading removed — the agent runs and the full trajectory is captured, but no criterion is checked, `weighted_score` is `None` (never `0.0`, which would be indistinguishable from "graded and scored zero"), and the row finalizes as **`FinalStatus.NOT_GRADED`**, whose `category` is a **fourth** bucket, `"ungraded"`. Ungraded rows leave BOTH sides of every rate: `RunSummary.pass_rate` / `error_share` and `VariantAggregate.pass_rate` divide by `tasks_graded` (`tasks_run - tasks_not_graded`), and `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter. **Only SUCCESS/FAILURE collapse into it** — `ERROR`, `TIMEOUT`, `BUILD_FAILED`, `MAX_TURNS_EXHAUSTED` and the budget stops are facts about the *run*, not about grading, and still apply (so `execute` still exits non-zero on a crash). The switch is `BatchRunConfig.grade` → `Orchestrator(grade=...)` → the **four** grading call sites (single-shot, evaluate-only, the simulation dialog check, and post-failure diagnostics); it crosses the docker boundary in `context.json` (defaulting to `True` in-container, so a host predating `execute` keeps grading). It is **deliberately not a task-config field** — no 5-layer merge, no `-D` path — because a task YAML must never declare itself ungraded; only the invoking command decides. `run` and `execute` share one body (`run_command.run_pipeline`) and differ solely in that flag, so there is no third code path. Two things are refused rather than degraded: `--junit-xml` (a report of verdicts, and there are none — though `reports_junit` still emits `` for an ungraded row it encounters) and simulation tasks (their turn-continuation logic reads criteria results, so an ungraded dialog would silently change its own stopping behavior). `stop_early:` blocks are inert under `execute` for the same reason the kill switch exists: the full trajectory is the deliverable. Motivating consumer: an external harness (Harbor / Terminal-Bench 2.0) that builds its own container, calls coder-eval as the agent, and grades with its own tests. -- **Detached grading (`evaluate` over a run dir) + `Sandbox.adopt`**: `coder-eval evaluate` takes two shapes, told apart by a **pure** resolver (`cli/evaluate_target.py`) on one probe — a target holding `task.json` is a run directory. Run-dir mode rebuilds the task from the run's own `task_config.resolved`, **not** by re-loading the YAML: `resolved` is post-merge, so variant overrides / `-D` / dataset expansion are already baked in and re-loading the source would silently grade a *different* task (fallback to `source_file` only when `resolved` no longer validates, and loudly). It seeds the fresh result from the prior one via `Orchestrator(prior_result=...)` → `_seed_from_prior_result`, which carries the trajectory (every derived figure — tokens, cost, `command_stats`, `model_used` — recomputes from `iterations`), `iteration_count`, execution facts, and **`early_stop` — load-bearing, because gate selection is FIRED-ONLY**: dropping it re-grades a truncated trajectory under the full-run strict-AND gate and can flip the verdict. Grader-host `environment_info` is preserved under a `graded_by` sub-dict rather than overwriting the run's. Two other parity fixes: `command_base_path` is now persisted into `environment_info` by `_sync_sandbox_command_path_with_agent` and restored in the evaluate-only branch (closing the PATH gap that method's docstring already named), and `_join_litellm_actual_cost` **skips** when `prior_result` is set (its join keys on a per-Orchestrator nonce the prior turns never carried, so it would clobber already-correct costs). The verdict is written back into the run's `task.json`, with the pre-grade record kept as `task.execute.json` — that in-place write is what makes plain `coder-eval aggregate ` rebuild a graded `run.json` with **zero** new code. **`Sandbox.adopt(workspace)`** is the grade-in-place primitive: it reuses `setup`'s adoption half but skips every *materializing* step (`_setup_template`, `_generate_cli_recorders`, venv/package installs, the destructive `$HOME` remediation), running only non-mutating derivation (mock-dir `+x`, venv *discovery*, plugin-tools pin); `_cleanup_on_exit` stays False so an adopted tree is never moved or deleted. In-place is **more correct**, not merely faster: `_setup_template` filters the copy through `_should_ignore_template_file`, which drops `node_modules` / `dist` / `build` / `.venv` / `.git`, so on the copy path a criterion like `test -f dist/bundle.js` fails as a *copying artifact* rather than as a verdict (verified: 0.00 "does not exist" on copy vs 1.00 in place). Defaults: in-place for a run dir, copy for a bare work dir (criteria can mutate it and it is the user's own tree); `--in-place`/`--copy` override. `adopt` hard-errors on `driver: docker` (a container workspace is unreachable from the host) and a re-grade refuses on a `reference_digest` mismatch. NOTE the Typer command is a thin wrapper over `run_evaluation(...)`, which has real Python defaults — calling a Typer command function in-process hands unspecified options an `OptionInfo` sentinel, which silently made `in_place=None` truthy. -- **`--resume` is command-relative**: `partition_for_resume(tasks, *, grade)` returns a four-way `ResumePartition` (`to_run` / `to_grade` / `prior_results` / `prior_resolved`), because **"finished" is not absolute — it depends on what the resuming command still owes the task**. A `NOT_GRADED` row carries a final status, so the original "has any final status" test called it complete: right for `execute --resume` (it finished executing), and wrong for `run --resume`, which was asked to grade and would instead report "already complete", grade nothing, and **exit 0**. Under `grade=True` those rows route to `to_grade`, where `_grade_resumed_tasks` runs the criteria against the trajectory and workspace already on disk via `orchestration/regrade.py::regrade_in_place` — reusing the agent spend, which is the entire reason `execute` and `run` are separate. The carve-out is **only** for `NOT_GRADED`: `FAILURE`/`ERROR` stay complete under both commands (resume has never retried failures — delete the task.json), and `clear_rerun_artifacts` deliberately skips `to_grade`, whose artifacts are the very thing being graded. A per-task grading failure is warned and folded back in with its ORIGINAL ungraded result, so one bad row neither aborts the resume nor vanishes from run.json. `grade` is in `_FINGERPRINT_DIFF_EXEMPT` because `execute` → `run --resume` is a supported flow, not config drift — and the warning's "already-finalized tasks keep their original-config results" text is actively wrong for it. **`orchestration/regrade.py` is the single implementation** shared by that path and `evaluate`'s run-dir mode (two copies would drift into two verdicts for the same run); it raises plain `RegradeError`, which the CLI wraps, since `orchestration/` must not import the CLI layer (CE004). One fidelity rule it enforces: a re-graded row keeps the **agent run's** `started_at`/`duration_seconds`, not the grading pass's — a 10-minute run re-graded in 2s would otherwise report 2s into `average_duration`, the report tables and the evalboard; the grading cost is preserved separately as `environment_info["grading_duration_seconds"]`. +- **Detached grading (`evaluate` over a run dir) + `Sandbox.adopt`**: `coder-eval evaluate` takes two shapes, told apart by a **pure** resolver (`cli/evaluate_target.py`) on one probe — a target holding `task.json` is a run directory. Run-dir mode rebuilds the task from the run's own `task_config.resolved`, **not** by re-loading the YAML: `resolved` is post-merge, so variant overrides / `-D` / dataset expansion are already baked in and re-loading the source would silently grade a *different* task (fallback to `source_file` only when `resolved` no longer validates, and loudly). It seeds the fresh result from the prior one via `Orchestrator(prior_result=...)` → `_seed_from_prior_result`, which carries the trajectory (every derived figure — tokens, cost, `command_stats`, `model_used` — recomputes from `iterations`), `iteration_count`, execution facts, and **`early_stop` — load-bearing, because gate selection is FIRED-ONLY**: dropping it re-grades a truncated trajectory under the full-run strict-AND gate and can flip the verdict. Carrying it is only half the fix — **both** grading paths select the gate through the single `Orchestrator._select_gate()`; the evaluate-only branch a detached grade actually takes originally called `all_criteria_passed` inline, so the seeded field was written and never read. `tests/test_seed_from_prior_result.py` partitions every `EvaluationResult` field as CARRIED or RECOMPUTED and fails closed on a new one, and asserts the two `_select_gate()` call sites. A prior status that `FinalStatus.is_execution_fact` (TIMEOUT / ERROR / BUILD_FAILED / the budget stops) is **preserved**, never overwritten: grading may only move `NOT_GRADED` to SUCCESS/FAILURE, since it neither repeated nor observed the agent phase. Grader-host `environment_info` is preserved under a `graded_by` sub-dict rather than overwriting the run's. Two other parity fixes: `command_base_path` is now persisted into `environment_info` by `_sync_sandbox_command_path_with_agent` and restored in the evaluate-only branch (closing the PATH gap that method's docstring already named), and `_join_litellm_actual_cost` **skips** when `prior_result` is set (its join keys on a per-Orchestrator nonce the prior turns never carried, so it would clobber already-correct costs). The verdict is written back into the run's `task.json`, with the pre-grade record kept as `task.execute.json` — that in-place write is what makes plain `coder-eval aggregate ` rebuild a graded `run.json` with **zero** new code. **`Sandbox.adopt(workspace)`** is the grade-in-place primitive: it reuses `setup`'s adoption half but skips every *materializing* step (`_setup_template`, `_generate_cli_recorders`, venv/package installs, the destructive `$HOME` remediation), running only non-mutating derivation (mock-dir `+x`, venv *discovery*, plugin-tools pin); `_cleanup_on_exit` stays False so an adopted tree is never moved or deleted, and `Sandbox.was_adopted` is set — the Orchestrator reads it to SKIP the `pre_run`/`post_run` hooks (`run()` calls them unconditionally with `cwd = sandbox_dir`, and several in-tree tasks stage fixtures there with `cp -a /app/[!.]* "$PWD/"`, which would overwrite the agent's deliverables before the criteria read them) and to KEEP `sandbox_path` in the `PreservationMode.NONE` cleanup arm (an adopted tree survives cleanup, so the path is not stale). The hooks' recorded results are carried from the prior run instead. In-place is **more correct**, not merely faster: `_setup_template` filters the copy through `_should_ignore_template_file`, which drops `node_modules` / `dist` / `build` / `.venv` / `.git`, so on the copy path a criterion like `test -f dist/bundle.js` fails as a *copying artifact* rather than as a verdict (verified: 0.00 "does not exist" on copy vs 1.00 in place). Defaults: in-place for a run dir, copy for a bare work dir (criteria can mutate it and it is the user's own tree); `--in-place`/`--copy` override. `adopt` hard-errors on `driver: docker` (a container workspace is unreachable from the host) and a re-grade refuses on a `reference_digest` mismatch — the digest is persisted into `environment_info` at staging time by `_stage_reference` (it shipped once as a read with no writer anywhere, so the guard was dead code), and `verify_reference_unchanged` now takes the task file it resolves against and RAISES on a vanished or unresolvable reference instead of returning silently. NOTE the Typer command is a thin wrapper over `run_evaluation(...)`, which has real Python defaults — calling a Typer command function in-process hands unspecified options an `OptionInfo` sentinel, which silently made `in_place=None` truthy. +- **`--resume` is command-relative**: `partition_for_resume(tasks, *, grade)` returns a four-way `ResumePartition` (`to_run` / `to_grade` / `prior_results` / `prior_resolved`), because **"finished" is not absolute — it depends on what the resuming command still owes the task**. A `NOT_GRADED` row carries a final status, so the original "has any final status" test called it complete: right for `execute --resume` (it finished executing), and wrong for `run --resume`, which was asked to grade and would instead report "already complete", grade nothing, and **exit 0**. Under `grade=True` those rows route to `to_grade`, where `_grade_resumed_tasks` runs the criteria against the trajectory and workspace already on disk via `orchestration/regrade.py::regrade_in_place` — reusing the agent spend, which is the entire reason `execute` and `run` are separate. The carve-out is **only** for `NOT_GRADED`: `FAILURE`/`ERROR` stay complete under both commands (resume has never retried failures — delete the task.json), and `clear_rerun_artifacts` deliberately skips `to_grade`, whose artifacts are the very thing being graded. A per-task grading failure is warned, STAMPED onto the folded-back row's `error_message` (the console line alone is not durable), and folded back in with its ORIGINAL ungraded result, so one bad row neither aborts the resume nor vanishes from run.json — and the exit gate counts `tasks_not_graded` **when `grade` is True**, so a `run` that graded nothing exits non-zero instead of telling CI the suite is fine. Under `execute` an ungraded row is the expected outcome and never fails the command. `grade` is in `_FINGERPRINT_DIFF_EXEMPT` because `execute` → `run --resume` is a supported flow, not config drift — and the warning's "already-finalized tasks keep their original-config results" text is actively wrong for it. **`orchestration/regrade.py` is the single implementation** shared by that path and `evaluate`'s run-dir mode, which DELEGATES to `regrade_in_place` rather than restating it (it originally hand-built its own Sandbox + Orchestrator and had already drifted — hardcoding `replicate_index=0`, so every replicate but the first was relabelled — which is exactly how two copies become two verdicts for the same run); it raises plain `RegradeError`, which the CLI wraps, since `orchestration/` must not import the CLI layer (CE004). One fidelity rule it enforces: a re-graded row keeps the **agent run's** `started_at`/`duration_seconds`, not the grading pass's — a 10-minute run re-graded in 2s would otherwise report 2s into `average_duration`, the report tables and the evalboard; the grading cost is preserved separately as `environment_info["grading_duration_seconds"]`. - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `stop_early.decide_within`, deliberately lives on `LiveSuccessCriterion` instead (see below) — the watcher must attribute a decision-step timeout to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express. - **Early stop on criterion (opt-in, per-criterion arming)**: a `stop_early:` block (`StopEarlyPolicy`) on a criterion ends a single-shot run early once the run's **armed** criteria decide the outcome, so a raised `max_turns` isn't wasted on the smoke flavor. The block's PRESENCE is the arming and alone activates the watcher — there is **no run-level master switch**: `run_limits.stop_early: false` is the run-level KILL SWITCH that force-disarms every block (the one-line experiment-variant/`-D` override for an authoritative full run), and `run_limits.stop_early: true` (the removed master arm) is a hard `EarlyStopConfigError` at resolution. The block exists on `LiveSuccessCriterion` only (currently `skill_triggered`, `command_executed` — so arming an unobservable criterion is unrepresentable, a pydantic extra-forbid error). Arming carries one implicit trigger (a native live-fail may fail-stop the run); its keys refine it: `on_pass: stop` (pass-stop the moment the criterion live-passes; default `continue` just latches) and `decide_within: N` (still undecided after N tool-call steps latches an **effective fail**, fed through the same fail-stop rule, reported as `decision_budget_exceeded` — an ordinary weighted fail, NOT a gate-bypassing force-fail; cumulative across retry attempts of the same turn). A trigger whose polarity the instance can't decide (per the abstract, checker-independent `live_decidable_polarities()`, a pure function of the criterion's own fields, paired with the checker's `live_verdict` override by lint rule CE025, a registry-based whole-tree check) is **inert by design** — one dataset-fanned YAML line serves both positive rows (pass/timeout live) and distractor rows (fail live). Verdicts **latch**: once a criterion decides, its `live_verdict` is never polled again. Stop rule is weighted, not strict-boolean: `run_limits.stop_early_gate_threshold` (default `1.0`, reproducing strict-AND behavior exactly) is the minimum weighted score (`Σ weight·score / Σ weight` over the armed subset) required to pass; a fail-stop fires once the armed set's **ceiling** (best case for everything still undecided) can no longer reach the threshold — so a low-weight fail or timeout that can't doom the gate is absorbed and the run continues — and is **deferred while any pass-capable armed criterion is undecided** (a distractor misfire never truncates a positive row's recall signal); a pass-stop fires once the `on_pass: stop` subset's **floor** (worst case) already meets the threshold, and is symmetrically **deferred while any pass-capable armed criterion outside the `on_pass: stop` subset is undecided** (so an early pass never freezes a sibling `on_pass: continue` criterion's signal out of the trajectory). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a kill-switched (`stop_early: false`) run. Driven by `orchestration/early_stop.py::EarlyStopWatcher` (built when `early_stop_active(task)`: ≥1 armed criterion, kill switch not thrown) through the agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. Gating is **FIRED-ONLY**: a run the watcher actually cut gates on the **armed subset** via the weighted `EvaluationResult.armed_criteria_passed`; a run that completes naturally — armed or not — gates strict-AND via `all_criteria_passed`, so adding a block never changes the verdict of a run it didn't cut. Note the gate keys on the watcher having FIRED (`result.early_stop is not None`), not on confirmed truncation — an agent that ignores `should_stop`, or a stop firing on the final message, still gates armed-only. Every resolution-time guardrail violation is a hard error at resolution (plan *and* run); the one load-time case — a `stop_early:` block on a non-live criterion — is a pydantic schema error at task load, which the run surface reports as a skipped task like any other malformed task. A runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo` (incl. `gate_threshold` at stop time), report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. No blocks anywhere ⇒ behavior byte-for-byte unchanged. diff --git a/evalboard/app/runs/[id]/run-view.tsx b/evalboard/app/runs/[id]/run-view.tsx index a5171fe5..40374b03 100644 --- a/evalboard/app/runs/[id]/run-view.tsx +++ b/evalboard/app/runs/[id]/run-view.tsx @@ -38,6 +38,9 @@ export interface RunMetrics { passed: number; failed: number; errored: number; + // Rows that ran but were never scored (`coder-eval execute`). Excluded from + // both sides of `pct`, so a fully ungraded run reports 0 of 0, not 0%. + ungraded: number; failedTotal: number; pct: number; // Per-task view of pass rate for repeated runs: distinct task_ids, and how @@ -72,6 +75,7 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics { let passed = 0; let failed = 0; let errored = 0; + let ungraded = 0; let cost = 0; let durationSum = 0; const costSamples: number[] = []; @@ -80,6 +84,11 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics { const cat = statusCategory(t.status); if (cat === "passed") passed++; else if (cat === "error") errored++; + // An ungraded row (`coder-eval execute`) was never scored. It leaves + // BOTH sides of the rate — the `else failed++` below would otherwise + // count it as a failure AND keep it in the denominator, rendering a + // clean execute run as 0% pass, N failed. + else if (cat === "ungraded") ungraded++; else failed++; if (t.matureSkipped) continue; if (t.totalCostUsd != null) { @@ -91,13 +100,15 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics { durSamples.push(t.durationSeconds); } } + const graded = total - ungraded; return { total, passed, failed, errored, + ungraded, failedTotal: failed + errored, - pct: total ? (passed / total) * 100 : 0, + pct: graded ? (passed / graded) * 100 : 0, ...(() => { // Per-task rollup (any replicate passed → task passed) via the shared // helper, so the run tile and the grid badge apply the same rule. diff --git a/evalboard/lib/__tests__/status.test.ts b/evalboard/lib/__tests__/status.test.ts new file mode 100644 index 00000000..a91db0a6 --- /dev/null +++ b/evalboard/lib/__tests__/status.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; + +import { + isGraded, + isPassStatus, + statusCategory, + statusSortRank, + type StatusCategory, +} from "../status"; + +// This module mirrors coder_eval's FinalStatus.category (models/enums.py), which +// is guarded there by `assert set(_STATUS_CATEGORIES) == set(FinalStatus)`. The +// mirror had no test at all, which is how NOT_GRADED came to be categorized as +// "unknown" and then counted as a failure by every rate helper downstream. +const EVERY_FINAL_STATUS: Record = { + SUCCESS: "passed", + FAILURE: "failed", + ERROR: "error", + BUILD_FAILED: "error", + TIMEOUT: "failed", + MAX_TURNS_EXHAUSTED: "failed", + TOKEN_BUDGET_EXCEEDED: "failed", + COST_BUDGET_EXCEEDED: "failed", + NOT_GRADED: "ungraded", +}; + +describe("statusCategory", () => { + it.each(Object.entries(EVERY_FINAL_STATUS))( + "maps %s to %s", + (status, expected) => { + expect(statusCategory(status)).toBe(expected); + }, + ); + + it("treats a missing status as unknown, distinct from ungraded", () => { + expect(statusCategory(null)).toBe("unknown"); + expect(statusCategory(null)).not.toBe(statusCategory("NOT_GRADED")); + }); + + it("does not classify an ungraded row as a pass or a failure", () => { + // The whole point of the fourth category: folding it into either side + // of a rate misreports a run that was never scored. + expect(statusCategory("NOT_GRADED")).not.toBe("passed"); + expect(statusCategory("NOT_GRADED")).not.toBe("failed"); + expect(isPassStatus("NOT_GRADED")).toBe(false); + }); +}); + +describe("isGraded", () => { + it("is false only for an ungraded row", () => { + expect(isGraded("NOT_GRADED")).toBe(false); + for (const status of Object.keys(EVERY_FINAL_STATUS)) { + if (status === "NOT_GRADED") continue; + expect(isGraded(status)).toBe(true); + } + // A null status is "no row here", not "ran but unscored". + expect(isGraded(null)).toBe(true); + }); +}); + +describe("statusSortRank", () => { + it("sorts failures first, ungraded in the middle, passes last", () => { + expect(statusSortRank("FAILURE")).toBeLessThan( + statusSortRank("NOT_GRADED"), + ); + expect(statusSortRank("NOT_GRADED")).toBeLessThan( + statusSortRank("SUCCESS"), + ); + }); +}); diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index ad31a97d..9da79c88 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -60,6 +60,7 @@ export interface RunSummary { tasksSucceeded: number; tasksFailed: number; tasksError: number; + tasksNotGraded: number; totalCostUsd: number | null; componentShas: ComponentSha[]; // What actually produced this run, for the run header. `harness` is the @@ -463,6 +464,9 @@ interface RawRunJson { tasks_succeeded?: number; tasks_failed?: number; tasks_error?: number; + // Rows that ran but were never scored (`coder-eval execute`). Optional: a + // run.json written before the field existed simply has none. + tasks_not_graded?: number; task_results?: RawTaskResult[]; // Values are scalars except `tool_plugins`, a {plugin: version} map of // the installed @uipath/*-tool packages (recorded since coder_eval #366). @@ -831,6 +835,7 @@ export async function readRunSummary( tasksSucceeded: data.tasks_succeeded ?? 0, tasksFailed: data.tasks_failed ?? 0, tasksError: data.tasks_error ?? 0, + tasksNotGraded: data.tasks_not_graded ?? 0, totalCostUsd: taskResults.length ? totalCost : null, componentShas: extractComponentShas(data.environment_info), harness: extractRunConfig(data).harness, diff --git a/evalboard/lib/status.ts b/evalboard/lib/status.ts index 17796f4c..7d261aa9 100644 --- a/evalboard/lib/status.ts +++ b/evalboard/lib/status.ts @@ -2,13 +2,16 @@ // Mirrors coder_eval `FinalStatus.category` (src/coder_eval/models/enums.py): // SUCCESS -> passed // ERROR / BUILD_FAILED -> error (BUILD_FAILED is an environment/setup failure) -// NOT_GRADED -> unknown (`coder-eval execute`: ran, deliberately unscored) +// NOT_GRADED -> ungraded (`coder-eval execute`: ran, deliberately unscored) // anything else (FAILURE, TIMEOUT, MAX_TURNS_EXHAUSTED, …) -> failed // -// NOT_GRADED maps to "unknown" rather than gaining a category of its own: every -// consumer already handles "unknown" (a null status) as "no verdict here", which -// is exactly what an ungraded row is. It is therefore not a pass, not a failure, -// and sorts in the middle — the same treatment a missing status gets. +// "ungraded" is its OWN member rather than being folded into "unknown". Folding +// it there looks safe — an ungraded row genuinely has no verdict — but every +// rate helper in this app is written as `if passed … else if error … else +// failed++`, so anything that is not a pass or an error is counted as a failure +// AND kept in the denominator. A clean `execute` run then renders as 0% pass, N +// failed. A distinct member makes that a type error at each site instead, so a +// consumer has to decide what to do with it. // // Note: this only categorizes coder_eval task statuses. UI status display // (e.g. StatusPill) also handles flow execution statuses like "Completed" @@ -16,16 +19,23 @@ import { taskVariantKey } from "./variants"; -export type StatusCategory = "passed" | "failed" | "error" | "unknown"; +export type StatusCategory = "passed" | "failed" | "error" | "ungraded" | "unknown"; export function statusCategory(status: string | null): StatusCategory { if (!status) return "unknown"; if (status === "SUCCESS") return "passed"; if (status === "ERROR" || status === "BUILD_FAILED") return "error"; - if (status === "NOT_GRADED") return "unknown"; + if (status === "NOT_GRADED") return "ungraded"; return "failed"; } +// Whether a row was measured at all. An ungraded row must leave BOTH sides of +// every rate — it is not a pass and not a failure, so counting it either way +// (or keeping it in a denominator) misreports a run that was never scored. +export function isGraded(status: string | null): boolean { + return statusCategory(status) !== "ungraded"; +} + // Whether a status is a pass (SUCCESS). The single predicate behind the // "a task passes if any replicate passed" rule. export function isPassStatus(status: string | null): boolean { diff --git a/evalboard/lib/trends.ts b/evalboard/lib/trends.ts index 24128a8e..2480fffc 100644 --- a/evalboard/lib/trends.ts +++ b/evalboard/lib/trends.ts @@ -12,6 +12,7 @@ import { } from "./overview"; import { DEFAULT_HARNESS } from "./harness"; import { DEFAULT_SOURCE, type Source } from "./sources"; +import { isGraded } from "./status"; import { taskCarriesRepoTag } from "./tags"; import type { ComponentSha } from "./runs"; @@ -151,7 +152,10 @@ export function aggregate(perRun: PerRun[]): TrendsData { } if (!b.skill && t.skill) b.skill = t.skill; for (const tg of t.tags) b.tagSet.add(tg); - b.totalCount += 1; + // An ungraded row (`coder-eval execute`) was never scored, so it + // enters neither side of the pass rate. Counting it in totalCount + // alone would drag a task's trend down as if it had failed. + if (isGraded(t.status)) b.totalCount += 1; if (t.matureSkipped) b.matureSkips += 1; b.statuses.push({ runId: id, diff --git a/evalboard/lib/watchlist.ts b/evalboard/lib/watchlist.ts index ae215881..79da9ebf 100644 --- a/evalboard/lib/watchlist.ts +++ b/evalboard/lib/watchlist.ts @@ -11,6 +11,7 @@ import type { PerRun } from "./overview"; import type { RunOverviewTask } from "./runs"; +import { isGraded } from "./status"; import { timeRatio } from "./timing"; import { turnRatio } from "./turns"; @@ -116,7 +117,9 @@ function stdev(xs: number[]): number { function skillPassSeq(runs: LoadedRun[], skill: string): number[] { const seq: number[] = []; for (const run of runs) { - const ts = run.tasks.filter((t) => t.skill === skill); + const ts = run.tasks.filter( + (t) => t.skill === skill && isGraded(t.status), + ); if (ts.length === 0) continue; seq.push(ts.filter((t) => isPass(t.status)).length / ts.length); } @@ -137,6 +140,8 @@ export function leaderboard(runs: LoadedRun[]): LeaderboardRow[] { for (const run of runs) { for (const t of run.tasks) { if (!t.skill) continue; + // Ungraded rows leave both sides of the rate — see isGraded. + if (!isGraded(t.status)) continue; total.set(t.skill, (total.get(t.skill) ?? 0) + 1); if (isPass(t.status)) passed.set(t.skill, (passed.get(t.skill) ?? 0) + 1); @@ -202,6 +207,8 @@ export function attention(runs: LoadedRun[]): AttentionRow[] { const ts = run.tasks.filter((t) => t.skill === skill); if (ts.length > 0) appeared++; for (const t of ts) { + // Ungraded rows leave both sides of the rate — see isGraded. + if (!isGraded(t.status)) continue; outcomes++; taskIds.add(t.taskId); if (isPass(t.status)) passes++; diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index 7de52fc3..8c61afc9 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -26,6 +26,7 @@ back_up_pre_grade_record, default_workspace, load_prior_result, + regrade_in_place, task_from_prior, verify_reference_unchanged, ) @@ -115,7 +116,7 @@ def _resolve_run_dir_or_work_dir(target: EvaluateTarget, workspace: Path | None) task.agent = parse_agent_config(**{**task.agent.model_dump(exclude_unset=True), "type": AgentKind.CLAUDE_CODE}) if prior is not None: - verify_reference_unchanged(prior, task) + verify_reference_unchanged(prior, task, task_file) return _ResolvedInputs( target=target, @@ -127,6 +128,18 @@ def _resolve_run_dir_or_work_dir(target: EvaluateTarget, workspace: Path | None) ) +def _replicate_index_of(run_dir: Path) -> int: + """Recover the replicate index a run directory encodes in its leaf name. + + Preservation lays runs out as ``///``. Hardcoding 0 + would relabel every replicate but the first as replicate 0. + """ + try: + return int(run_dir.name) + except ValueError: + return 0 + + def evaluate_command( task_or_run_dir: Path = typer.Argument( # noqa: B008 ..., @@ -261,6 +274,22 @@ def run_evaluation( sandbox = Sandbox(sandbox_config, task_id=task.task_id, task_dir=task_dir) async def _setup_and_run() -> EvaluationResult: + if grade_in_place and prior is not None: + # Delegate to the shared re-grade core. Restating its body here is + # how this path and `run --resume` came to differ (replicate_index, + # error semantics) while CLAUDE.md called regrade.py the single + # implementation — two copies of "how to re-grade" drift into two + # verdicts for the same run. + return await regrade_in_place( + task=task, + prior=prior, + workspace=graded_dir, + run_dir=prepared_run_dir, + task_file=task_file, + source_yaml=source_yaml, + variant_id=prior.variant_id, + replicate_index=_replicate_index_of(target.target), + ) if grade_in_place: await asyncio.to_thread(sandbox.adopt, graded_dir) else: diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index 65337c65..5c3de001 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -604,7 +604,14 @@ async def _run_all_tasks( flush_telemetry() # Exit with non-zero code if any tasks failed, errored, or any suite failed its thresholds. - if summary.tasks_failed > 0 or summary.tasks_error > 0 or failed_suite_gates > 0: + # + # An ungraded row counts too, but only under `run`: `run` was asked for a + # verdict and did not produce one (the grade crashed, or --resume could not + # grade the row), which is a failure of the command even though the row is + # neither `failed` nor `error`. Under `execute` an ungraded row is the + # expected outcome for every task, so it must not fail the command. + ungraded_but_asked_to_grade = grade and summary.tasks_not_graded > 0 + if summary.tasks_failed > 0 or summary.tasks_error > 0 or failed_suite_gates > 0 or ungraded_but_asked_to_grade: raise typer.Exit(1) @@ -686,7 +693,7 @@ async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[Resol for rt in to_grade: prior = load_prior_result(rt.run_dir) try: - verify_reference_unchanged(prior, rt.task) + verify_reference_unchanged(prior, rt.task, rt.task_file) workspace = default_workspace(rt.run_dir, prior) # Preserve the ungraded record BEFORE the orchestrator overwrites # task.json in this same directory. @@ -703,7 +710,12 @@ async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[Resol ) except (RegradeError, OSError, RuntimeError, ValueError) as e: console.print(f"[yellow]⚠[/] Could not grade {rt.task.task_id}: {e}") + # Stamp the reason onto the row. Without it the failure survives only + # in this console line: the folded-back result keeps the execute + # phase's empty error_message, so run.json, the reports and CI show + # an ungraded row with no explanation of why grading never happened. result = prior + result.error_message = f"Grading failed during --resume: {e}" graded.append( ( rt, diff --git a/src/coder_eval/isolation/docker_runner.py b/src/coder_eval/isolation/docker_runner.py index bb967222..3b46631b 100644 --- a/src/coder_eval/isolation/docker_runner.py +++ b/src/coder_eval/isolation/docker_runner.py @@ -871,8 +871,29 @@ async def _parse_result_or_raise(self, output_dir: Path, returncode: int, log_pa # rather than crashing with an uncaught ValidationError/JSONDecodeError. raise await self._handle_malformed_task_json(task_json, log_path, exc) from exc self._warn_on_version_mismatch(result) + self._assert_grade_honored(result) return result + def _assert_grade_honored(self, result: EvaluationResult) -> None: + """Fail loudly when `execute` came back with a graded verdict. + + ``grade`` crosses the boundary only through ``context.json``. An image + that predates ``execute`` ignores the unknown key and grades anyway, and + the image-version preflight only warns — so ``execute --driver docker`` + against a stale image would silently produce SUCCESS/FAILURE rows that + look like a normal graded run. Version skew must not change what a + command MEANS, so refuse the row rather than publish it. + """ + if self.grade or result.final_status.is_execution_fact: + return + if result.final_status is not FinalStatus.NOT_GRADED: + raise DockerRunError( + "`coder-eval execute` asked the container not to grade, but it returned " + + f"{result.final_status.value} with {len(result.success_criteria_results)} criterion " + + "result(s). The runtime image predates `execute` and ignored the request; " + + "rebuild or pull a matching agent image." + ) + async def _handle_malformed_task_json(self, task_json: Path, log_path: Path, exc: ValueError) -> DockerRunError: """Degrade a present-but-malformed task.json; return the DockerRunError to raise. diff --git a/src/coder_eval/models/enums.py b/src/coder_eval/models/enums.py index 7135de37..f4586b6a 100644 --- a/src/coder_eval/models/enums.py +++ b/src/coder_eval/models/enums.py @@ -33,6 +33,19 @@ def icon(self) -> str: """Single-character icon for reports and CLI output.""" return _STATUS_ICONS[self] + @property + def is_execution_fact(self) -> bool: + """True when this status records HOW THE RUN ENDED, not what grading decided. + + A detached grade (``evaluate `` / ``run --resume``) re-runs the + criteria over a trajectory it did not produce, so it may only move a row + between the three GRADING outcomes — ``NOT_GRADED`` -> ``SUCCESS`` / + ``FAILURE``. It must never launder a run that timed out, crashed, or blew + a budget into a pass: those statuses describe the agent phase, which the + grading pass neither repeated nor observed. + """ + return _EXECUTION_FACT_STATUSES[self] + # Every FinalStatus maps to exactly one reporting category, listed EXPLICITLY (no # catch-all default) so a newly-added status fails the assert below until it is @@ -76,6 +89,25 @@ def icon(self) -> str: assert set(_STATUS_ICONS) == set(FinalStatus), "Missing icon for FinalStatus member" +# Explicit, no catch-all, for the same reason as the two maps above: a new status +# must be classified as "the agent phase ended this way" (True — a detached grade +# preserves it) or "grading decided this" (False — a detached grade replaces it). +# Defaulting either way silently is how an ERROR row becomes a SUCCESS. +_EXECUTION_FACT_STATUSES: dict[FinalStatus, bool] = { + FinalStatus.SUCCESS: False, + FinalStatus.FAILURE: False, + FinalStatus.NOT_GRADED: False, + FinalStatus.ERROR: True, + FinalStatus.BUILD_FAILED: True, + FinalStatus.TIMEOUT: True, + FinalStatus.MAX_TURNS_EXHAUSTED: True, + FinalStatus.TOKEN_BUDGET_EXCEEDED: True, + FinalStatus.COST_BUDGET_EXCEEDED: True, +} + +assert set(_EXECUTION_FACT_STATUSES) == set(FinalStatus), "Unclassified FinalStatus member" + + class ApiBackend(StrEnum): """API backend for LLM calls.""" diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py index 4c28982b..c399d7bf 100644 --- a/src/coder_eval/models/experiment.py +++ b/src/coder_eval/models/experiment.py @@ -190,7 +190,11 @@ class VariantResult(BaseModel): # noqa: CE009 -- persisted result model; round- variant_id: str task_id: str - weighted_score: float + # None when nothing was graded (`coder-eval execute`), mirroring + # EvaluationResult.weighted_score. A plain float here would launder the + # ungraded None into 0.000, which renders as — and is picked as a best + # variant against — a real score of zero. + weighted_score: float | None = None final_status: FinalStatus duration_seconds: float total_tokens: int | None = None @@ -257,9 +261,14 @@ def _check_task_count_invariant(self) -> VariantAggregate: raise ValueError(f"Task count invariant violated: {total} != {self.tasks_run}") return self + @computed_field # type: ignore[prop-decorator] @property def tasks_graded(self) -> int: - """Tasks actually measured — ``pass_rate``'s denominator.""" + """Tasks actually measured — ``pass_rate``'s denominator. + + Serialized for the same reason as its RunSummary twin: a consumer that + cannot read the denominator re-derives the rate and drifts. + """ return self.tasks_run - self.tasks_not_graded @computed_field # type: ignore[prop-decorator] diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 170b868b..770a1bb7 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -887,7 +887,11 @@ class SuiteRollup(BaseModel): rows_passed: int rows_failed: int rows_error: int - pass_rate: float = Field(ge=0.0, le=1.0, description="rows_passed / rows_total") + # The fourth bucket, matching RunSummary.tasks_not_graded and + # VariantAggregate.tasks_not_graded. Defaulted so a suite.json written before + # `execute` existed still parses. + rows_not_graded: int = 0 + pass_rate: float = Field(ge=0.0, le=1.0, description="rows_passed / rows_graded (ungraded rows excluded)") average_weighted_score: float | None = Field( default=None, description="Mean weighted_score across rows that produced one." ) @@ -911,6 +915,20 @@ class SuiteRollup(BaseModel): ), ) + @model_validator(mode="after") + def _check_row_count_invariant(self) -> SuiteRollup: + """The same guard RunSummary carries, which this model was missing. + + Without it a row that lands outside all four buckets — the shape a new + FinalStatus category takes before every counter is updated — silently + drops out of the rollup instead of failing. + """ + buckets = self.rows_passed + self.rows_failed + self.rows_error + self.rows_not_graded + if buckets != self.rows_total: + total = f"{self.rows_passed} + {self.rows_failed} + {self.rows_error} + {self.rows_not_graded}" + raise ValueError(f"Suite row count invariant violated: {total} != {self.rows_total}") + return self + class SkippedTask(BaseModel): """A task YAML that was excluded from the run before reaching the orchestrator. @@ -1030,10 +1048,13 @@ def eval_result_total_cost(result: EvaluationResult) -> float | None: class RunSummary(BaseModel): """Summary of an entire evaluation run across multiple tasks. - ``pass_rate`` is ``tasks_succeeded / tasks_run``: every dispatched task is in the - denominator, errors included as misses. The previous formula excluded errors, - which paid a bonus for erroring. ``error_share`` reports how much of the rate is - errors, so a bad infrastructure night shows instead of being absorbed. + ``pass_rate`` is ``tasks_succeeded / tasks_graded``: every task that was + MEASURED is in the denominator, errors included as misses. An earlier formula + excluded errors, which paid a bonus for erroring. ``error_share`` reports how + much of the rate is errors, so a bad infrastructure night shows instead of + being absorbed. Only ungraded tasks (``coder-eval execute``) leave the + denominator — they were never measured, so a 0/0 run has no rate at all + rather than a 0% one. This is the framework's single denominator: every reporting surface reads ``pass_rate`` rather than re-deriving one. Derived metrics here are computed, @@ -1114,9 +1135,17 @@ def _check_task_count_invariant(self) -> RunSummary: raise ValueError(f"Task count invariant violated: {total} != {self.tasks_run}") return self + @computed_field @property def tasks_graded(self) -> int: - """Tasks that were actually measured — the denominator for every rate below.""" + """Tasks that were actually measured — the denominator for every rate below. + + A ``computed_field`` rather than a plain property so it reaches run.json: + it is the denominator of `pass_rate` and `error_share`, and a consumer + that cannot read it has to re-derive the rate from the raw counts — which + is precisely how a consumer ends up publishing a different number for the + same run. REPORT_SCHEMA.md documents it as serialized. + """ return self.tasks_run - self.tasks_not_graded # Derived run metrics: computed_fields over the stored counts and diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index 3e8c1500..24d0effe 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -831,7 +831,7 @@ def _pick_worst_status(statuses: list[FinalStatus]) -> FinalStatus: def _mean_graded_score(vr_list: list[VariantResult]) -> float: """Mean ``weighted_score`` over the graded rows; ``0.0`` when none were graded.""" - graded = [v.weighted_score for v in vr_list if v.final_status.category != "ungraded"] + graded = [v.weighted_score for v in vr_list if v.weighted_score is not None] return sum(graded) / len(graded) if graded else 0.0 @@ -878,11 +878,15 @@ def aggregate_results( # Collect per-replicate scores keyed variant_id → task_id → [scores] for stats rendering. per_replicate_scores: dict[str, dict[str, list[float]]] = {} for (task_id, variant_id), reps in task_variant_reps.items(): - per_replicate_scores.setdefault(variant_id, {})[task_id] = [r.result.weighted_score or 0.0 for r in reps] + per_replicate_scores.setdefault(variant_id, {})[task_id] = [ + r.result.weighted_score for r in reps if r.result.weighted_score is not None + ] task_variants: dict[str, list[VariantResult]] = {} for (task_id, variant_id), reps in task_variant_reps.items(): - scores = [r.result.weighted_score or 0.0 for r in reps] + # Ungraded replicates drop out entirely rather than contributing 0.0 — + # `or 0.0` would average a clean `execute` run down to a real-looking zero. + scores = [r.result.weighted_score for r in reps if r.result.weighted_score is not None] non_errored = [r for r in reps if r.result.final_status.category != "error"] durations = [r.result.duration_seconds for r in non_errored] statuses = [r.result.final_status for r in reps] @@ -895,7 +899,7 @@ def aggregate_results( variant_result = VariantResult( variant_id=variant_id, task_id=task_id, - weighted_score=sum(scores) / len(scores), + weighted_score=sum(scores) / len(scores) if scores else None, final_status=final_status, duration_seconds=sum(durations), total_tokens=sum(token_vals) if token_vals else None, @@ -910,9 +914,12 @@ def aggregate_results( # Build task summaries task_summaries: list[TaskExperimentSummary] = [] for task_id, variants in task_variants.items(): - best = max(variants, key=lambda v: (v.weighted_score, v.variant_id)) - scores = [v.weighted_score for v in variants] - top_count = sum(1 for v in variants if v.weighted_score == best.weighted_score) + # Only graded variants can win or set a spread. Including ungraded ones + # at 0.0 would name an arbitrary "best" among scores that do not exist. + scored = [(v, v.weighted_score) for v in variants if v.weighted_score is not None] + best = max(scored, key=lambda pair: (pair[1], pair[0].variant_id))[0] if scored else variants[0] + scores = [s for _, s in scored] + top_count = sum(1 for _, s in scored if s == best.weighted_score) rep_counts = {v.replicate_count for v in variants} task_summaries.append( TaskExperimentSummary( @@ -920,7 +927,7 @@ def aggregate_results( variant_results=variants, best_variant=best.variant_id, is_tie=top_count > 1, - score_spread=max(scores) - min(scores), + score_spread=(max(scores) - min(scores)) if scores else 0.0, replicate_count=min(rep_counts) if rep_counts else 1, ) ) diff --git a/src/coder_eval/orchestration/regrade.py b/src/coder_eval/orchestration/regrade.py index aa6b15c7..8eba319d 100644 --- a/src/coder_eval/orchestration/regrade.py +++ b/src/coder_eval/orchestration/regrade.py @@ -20,7 +20,7 @@ import logging from pathlib import Path -from coder_eval.models import EvaluationResult, PreservationMode, TaskDefinition +from coder_eval.models import EvaluationResult, PreservationMode, TaskConfigRecord, TaskDefinition from coder_eval.sandbox import Sandbox @@ -61,8 +61,6 @@ def task_from_prior(prior: EvaluationResult, run_dir: Path) -> tuple[TaskDefinit schema change since the run), and says so loudly — a quiet fallback would reintroduce exactly the drift above. """ - from .task_loader import load_task - record = prior.task_config if record is None: raise RegradeError( @@ -70,21 +68,55 @@ def task_from_prior(prior: EvaluationResult, run_dir: Path) -> tuple[TaskDefinit + "rebuilt. Pass the task file explicitly: coder-eval evaluate " ) try: - return TaskDefinition.model_validate(record.resolved), record.source_yaml + task = TaskDefinition.model_validate(record.resolved) except ValueError as e: - if not record.source_file or not Path(record.source_file).is_file(): - raise RegradeError( - f"The resolved task config in {run_dir / TASK_JSON} no longer validates ({e}), and " - + "its source YAML is unavailable. Pass the task file explicitly." - ) from e - logger.warning( - "The recorded resolved config does not validate (%s); falling back to %s. Variant " - + "overrides, -D flags and dataset expansion from the original run are NOT reapplied, " - + "so this grade may not match what ran.", - e, - record.source_file, - ) - return load_task(Path(record.source_file)) + return _fall_back_to_source(record, run_dir, e) + warn_on_embedded_commands(task, run_dir) + return task, record.source_yaml + + +def warn_on_embedded_commands(task: TaskDefinition, run_dir: Path) -> None: + """Name the shell commands a rebuilt config will execute on this host. + + ``task_config.resolved`` is data that travels inside a run directory, and a + run directory is a shareable artifact — the detached-grading flow exists so + one machine can execute and another can grade. Rebuilding the task from it + means the *run dir* decides what ``run_command`` criteria the grader runs, + with the grader's environment. That is the intended behavior (it is how the + grade reproduces the executed config), but it must not be invisible: print + what will run so an unexpected command is noticed before it executes. + """ + commands = [cmd for c in task.success_criteria if isinstance(cmd := getattr(c, "command", None), str)] + commands += [c.command for c in task.pre_run] + [c.command for c in task.post_run] + if not commands: + return + logger.warning( + "Grading %s runs %d shell command(s) taken from that run's own recorded config: %s", + run_dir, + len(commands), + "; ".join(commands), + ) + + +def _fall_back_to_source(record: TaskConfigRecord, run_dir: Path, e: ValueError) -> tuple[TaskDefinition, str]: + """The loud source-YAML fallback for a resolved config that no longer validates.""" + from .task_loader import load_task + + if not record.source_file or not Path(record.source_file).is_file(): + raise RegradeError( + f"The resolved task config in {run_dir / TASK_JSON} no longer validates ({e}), and " + + "its source YAML is unavailable. Pass the task file explicitly." + ) from e + logger.warning( + "The recorded resolved config does not validate (%s); falling back to %s. Variant " + + "overrides, -D flags and dataset expansion from the original run are NOT reapplied, " + + "so this grade may not match what ran.", + e, + record.source_file, + ) + task, source_yaml = load_task(Path(record.source_file)) + warn_on_embedded_commands(task, run_dir) + return task, source_yaml def default_workspace(run_dir: Path, prior: EvaluationResult) -> Path: @@ -112,23 +144,44 @@ def default_workspace(run_dir: Path, prior: EvaluationResult) -> Path: return children[0] if len(children) == 1 else artifacts -def verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition) -> None: +def verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition, task_file: Path | None) -> None: """Refuse to grade when the reference tree changed since the run. ``reference_comparison`` and reference-carrying judges score against ``task.reference.directory``. If it moved since the run, the re-grade would silently measure the agent's old work against a new answer key. + + ``task_file`` is what ``reference.directory`` resolves against, so it is + required for any task that declares one — resolving without it raises, which + is why it is threaded through rather than passed as ``None``. """ + if task.reference is None: + return recorded = prior.environment_info.get("reference_digest") - if not isinstance(recorded, str) or task.reference is None: + if not isinstance(recorded, str): + # A run that predates the digest being persisted. Say so: silence here is + # what made this whole guard dead code for its first release. + logger.warning( + "This run recorded no reference_digest, so the answer key cannot be verified. " + + "Grading proceeds; a reference edited since the run would go undetected." + ) return from coder_eval.path_utils import digest_tree from .evaluation import resolve_reference_dir - resolved = resolve_reference_dir(task, None) + try: + resolved = resolve_reference_dir(task, task_file) + except (FileNotFoundError, ValueError) as e: + raise RegradeError( + f"This run's task declares a reference directory that cannot be resolved now ({e}), " + + "so its contents cannot be verified against the executed run." + ) from e if resolved is None or not resolved.is_dir(): - return + raise RegradeError( + f"The reference directory recorded for this run is gone ({resolved}). Grading now " + + "would score against a missing answer key. Restore it, or re-run the task." + ) if digest_tree(resolved) != recorded: raise RegradeError( f"The reference directory {resolved} changed since this run was executed " diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index e9c991ea..1e5c2540 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -2,6 +2,7 @@ import asyncio import logging +import os import re import tempfile import time @@ -608,7 +609,21 @@ def _kill_agent_subprocess_sync() -> None: # (like TIMEOUT / BUILD_FAILED / the budget stops on the except # branches below) is a fact about the RUN, not about grading, and # still applies. With grade=True the chain is unchanged. - if success: + # + # A detached grade goes further: the prior run's terminal status + # may itself be an execution fact (TIMEOUT, ERROR, a budget stop) + # that this pass neither repeated nor observed, so grading must + # not overwrite it. Without this, a crashed run re-graded against + # its half-finished workspace reports SUCCESS — with the original + # error_message still attached. + inherited = self.prior_result.final_status if self.prior_result is not None else None + if inherited is not None and inherited.is_execution_fact: + logger.info( + "Preserving the run's terminal status %s: grading cannot overturn an execution fact.", + inherited.value, + ) + self.result.final_status = inherited + elif success: self.result.final_status = FinalStatus.SUCCESS elif self.result.max_turns_exhausted: self.result.final_status = FinalStatus.MAX_TURNS_EXHAUSTED @@ -767,7 +782,22 @@ def _seed_from_prior_result(self) -> None: self.result.max_turns_exhausted = prior.max_turns_exhausted self.result.error_message = prior.error_message self.result.error_details = prior.error_details + self.result.error_log_tail = prior.error_log_tail self.result.sdk_options = prior.sdk_options + self.result.agent_config = prior.agent_config + self.result.expected_commands = prior.expected_commands + self.result.simulation = prior.simulation + + # The hooks belong to the execute phase and are NOT re-run against an + # adopted workspace (see _skip_hooks_for_adopted), so their recorded + # outcomes would otherwise vanish from the graded row. + self.result.pre_run_results = list(prior.pre_run_results) + self.result.post_run_results = list(prior.post_run_results) + + # The artifacts pointer. An adopted sandbox is not deleted by cleanup(), + # so the path stays valid — and a SECOND grade needs it, since without it + # the caller falls back to guessing the workspace. + self.result.sandbox_path = prior.sandbox_path # environment_info: the prior run's capture describes the machine that # RAN the task (installed_tools, api route, coder_eval version). Ours @@ -1325,6 +1355,12 @@ async def _stage_reference(self) -> None: destination = staging / "reference" self._reference_dir = await asyncio.to_thread(stage_reference_dir, source, destination) self._reference_digest = await asyncio.to_thread(digest_tree, self._reference_dir) + # Persist it: a DETACHED grade happens in a different process with no + # access to this instance, and refuses to score old work against a new + # answer key by comparing the tree it stages against this recorded hash + # (orchestration/regrade.py::verify_reference_unchanged). + if self.result is not None: + self.result.environment_info["reference_digest"] = self._reference_digest self._validate_reference_consumers() def _validate_reference_consumers(self) -> None: @@ -1422,7 +1458,7 @@ async def _setup(self) -> None: # against ambient PATH and can disagree with the original verdict. restored_path = self.result.environment_info.get("command_base_path") if isinstance(restored_path, str) and restored_path: - self.sandbox.set_command_base_path(restored_path) + self.sandbox.set_command_base_path(self._sanitize_restored_path(restored_path)) self._resolve_routes() self._record_route_environment_info() @@ -2009,6 +2045,94 @@ def _accumulate_judge_usage( # forward so it isn't dropped from the latest results list. r.token_usage = prior + def _sanitize_restored_path(self, recorded: str) -> str: + """Filter a PATH restored from a run's own ``task.json`` before prepending it. + + The restored value is PREPENDED ahead of the host PATH, and it arrives + from a file inside the directory being graded — a run dir is a shareable + artifact (that is the whole point of the detached-grading flow), and under + ``driver: docker`` it is bind-mounted writable into the container the agent + runs in. Prepending it verbatim lets a run dir decide which binary + ``pytest`` resolves to on the grader's host. + + Two filters, both cheap and both about what PATH parity actually needs: + drop anything that is not an existing directory (a dead entry buys no + parity), and drop any entry inside the workspace being graded (that tree is + agent-writable, so a shim dropped there would shadow a real tool). What + remains is the run's genuine toolchain locations. + """ + workspace = self.sandbox.sandbox_dir.resolve() if self.sandbox and self.sandbox.sandbox_dir else None + kept: list[str] = [] + for entry in recorded.split(os.pathsep): + if not entry: + continue + candidate = Path(entry) + if not candidate.is_dir(): + logger.debug("Dropping recorded PATH entry %s: not a directory here.", entry) + continue + resolved = candidate.resolve() + if workspace is not None and (resolved == workspace or workspace in resolved.parents): + logger.warning( + "Dropping recorded PATH entry %s: it lies inside the workspace being graded, " + + "so a binary there could shadow a real tool on the grader's host.", + entry, + ) + continue + kept.append(str(resolved)) + return os.pathsep.join(kept) + + def _select_gate(self) -> bool: + """Apply the verdict gate to the criteria results already on ``self.result``. + + Gate selection is FIRED-ONLY: the weighted armed gate applies iff the + watcher actually cut the run (``early_stop is not None``) — on a truncated + trajectory the unarmed criteria never had the chance to be satisfied, so + they stay advisory. A run that completed naturally (armed or not, watcher + never fired or disarmed fail-open) has a full trajectory and gates + strict-AND over every gating criterion, exactly like an unarmed run — + arming a criterion (e.g. adding a ``decide_within`` fail-fast timeout) + must never change the verdict of a run it didn't cut. + + BOTH grading paths must call this. A detached grade (``evaluate + `` / ``run --resume``) reaches the verdict through the + evaluate-only branch, where ``early_stop`` arrives via + ``_seed_from_prior_result`` rather than from a live watcher; selecting + the gate there in a second, hand-written place is exactly how the + seeded field came to be carried but never read — re-grading an + early-stopped run under the full-run strict-AND gate flips its verdict. + """ + assert self.result is not None + if self.result.early_stop is not None: + # One gate for every early-stopped run, no per-reason branches: a + # decision-budget stop is just a fail-stop whose deciding criterion + # timed out (the watcher only fires once the weighted ceiling + # proves the armed gate cannot pass). The ceiling is an upper bound + # on the authoritative armed score only because the watcher reduces + # the SAME trajectory the checker scores — it records UNRESOLVED + # tool ends exactly like the agent's EventCollector does (see + # EarlyStopWatcher._on_event_impl) — so the weighted armed gate is + # correct whether the watcher fired on a pass, a fail, or a timeout. + gate_threshold = ( + self.task.run_limits.stop_early_gate_threshold + if self.task.run_limits is not None + else DEFAULT_STOP_EARLY_GATE_THRESHOLD + ) + armed_count = sum(1 for c in self.task.success_criteria if c.is_stop_armed) + logger.info( + "Early-stopped run (%s): gating on %d armed criteria (%d advisory, not gated).", + self.result.early_stop.reason.value, + armed_count, + len(self.task.success_criteria) - armed_count, + ) + return self.result.armed_criteria_passed(self.task.success_criteria, gate_threshold) + + if self._early_stop_watcher is not None: + if self._early_stop_watcher.disarmed: + logger.info("early-stop watcher disarmed fail-open (verdict error): gating on the full set.") + else: + logger.info("early-stop armed but never fired (run completed naturally): gating on the full set.") + return self.result.all_criteria_passed(self.task.success_criteria) + async def _evaluation_loop(self) -> bool: """Run the main evaluation loop. @@ -2058,7 +2182,7 @@ async def _evaluation_loop(self) -> bool: turn_records=self.result.iterations, ) self.result.success_criteria_results = criteria_results - return self.result.all_criteria_passed(self.task.success_criteria) + return self._select_gate() # Working directory context prepended to every prompt (including feedback). # The agent resumes its session between iterations via session_id. @@ -2127,44 +2251,7 @@ async def _evaluation_loop(self) -> bool: pairs = list(zip(criteria_results, self.task.success_criteria, strict=True)) passed_count = sum(1 for r, c in pairs if r.score >= c.pass_threshold) total_count = len(pairs) - # Gate selection is FIRED-ONLY: the weighted armed gate applies iff the - # watcher actually cut the run (early_stop is not None) — on a truncated - # trajectory the unarmed criteria never had the chance to be satisfied, - # so they stay advisory. A run that completed naturally (armed or not, - # watcher never fired or disarmed fail-open) has a full trajectory and - # gates strict-AND over every gating criterion, exactly like an unarmed - # run — arming a criterion (e.g. adding a decide_within fail-fast - # timeout) must never change the verdict of a run it didn't cut. - if self.result.early_stop is not None: - # One gate for every early-stopped run, no per-reason branches: a - # decision-budget stop is just a fail-stop whose deciding criterion - # timed out (the watcher only fires once the weighted ceiling - # proves the armed gate cannot pass). The ceiling is an upper bound - # on the authoritative armed score only because the watcher reduces - # the SAME trajectory the checker scores — it records UNRESOLVED - # tool ends exactly like the agent's EventCollector does (see - # EarlyStopWatcher._on_event_impl) — so the weighted armed gate is - # correct whether the watcher fired on a pass, a fail, or a timeout. - gate_threshold = ( - self.task.run_limits.stop_early_gate_threshold - if self.task.run_limits is not None - else DEFAULT_STOP_EARLY_GATE_THRESHOLD - ) - all_passed = self.result.armed_criteria_passed(self.task.success_criteria, gate_threshold) - armed_count = sum(1 for c in self.task.success_criteria if c.is_stop_armed) - logger.info( - "Early-stopped run (%s): gating on %d armed criteria (%d advisory, not gated).", - self.result.early_stop.reason.value, - armed_count, - total_count - armed_count, - ) - else: - if self._early_stop_watcher is not None: - if self._early_stop_watcher.disarmed: - logger.info("early-stop watcher disarmed fail-open (verdict error): gating on the full set.") - else: - logger.info("early-stop armed but never fired (run completed naturally): gating on the full set.") - all_passed = self.result.all_criteria_passed(self.task.success_criteria) + all_passed = self._select_gate() # Reuse the model method for weighted score (single source of truth) self.result.calculate_weighted_score(self.task.success_criteria) @@ -2835,8 +2922,10 @@ async def _run_pre_run_commands(self) -> None: outer ``except Exception`` handler and lands the run as ``FinalStatus.ERROR``. Post-run commands and cleanup still execute via the ``finally`` block. + + Skipped entirely on an adopted sandbox — see ``_skip_hooks_for_adopted``. """ - if self.result is None: + if self.result is None or self._skip_hooks_for_adopted("pre_run"): return await self._run_command_list(self.task.pre_run, self.result.pre_run_results, "pre_run") @@ -2846,11 +2935,39 @@ async def _run_post_run_commands(self) -> None: See ``_run_command_list``. Post-run commands are informational only — ``fail_on_error`` is not part of ``PostRunCommand``, so failures are warning-logged and never affect the evaluation verdict. + + Skipped entirely on an adopted sandbox — see ``_skip_hooks_for_adopted``. """ - if self.result is None: + if self.result is None or self._skip_hooks_for_adopted("post_run"): return await self._run_command_list(self.task.post_run, self.result.post_run_results, "post_run") + def _skip_hooks_for_adopted(self, phase: str) -> bool: + """True when ``phase``'s commands must not run against an adopted sandbox. + + ``adopt()`` guarantees it materializes nothing into the workspace, but + that guarantee is only as strong as its weakest caller: ``run()`` invokes + the pre/post-run hooks unconditionally, and those commands run with + ``cwd = sandbox_dir``. Several in-tree tasks stage fixtures there + (``cp -a /app/[!.]* "$PWD/"``), so re-running them during a detached + grade would overwrite the agent's deliverables *before* the criteria read + them — silently changing the verdict and destroying preserved artifacts. + + The hooks belong to the EXECUTE phase; the prior run already ran them, + and their recorded results are carried over by ``_seed_from_prior_result``. + """ + if self.sandbox is None or not self.sandbox.was_adopted: + return False + commands = self.task.pre_run if phase == "pre_run" else self.task.post_run + if commands: + logger.info( + "Skipping %d %s command(s): the sandbox was adopted for grading, and re-running them " + + "would mutate the workspace under evaluation.", + len(commands), + phase, + ) + return True + async def _cleanup(self) -> None: """Clean up all resources.""" # Stop agent @@ -2914,8 +3031,15 @@ async def _cleanup(self) -> None: await asyncio.to_thread(self.sandbox.grant_read_access) logger.info(f"Sandbox preserved (in-place): {self.sandbox.sandbox_dir}") elif self.preservation_mode == PreservationMode.NONE and self.result: - # Sandbox will be deleted by cleanup() below; clear stale path. - self.result.sandbox_path = None + if self.sandbox.was_adopted: + # An adopted sandbox belongs to the caller and survives + # cleanup(), so the path is not stale — and clearing it + # would strip the graded row of its artifacts pointer + # (which a second grade needs to find the workspace). + self.result.sandbox_path = str(self.sandbox.sandbox_dir) + else: + # Sandbox will be deleted by cleanup() below; clear stale path. + self.result.sandbox_path = None elif self.result: # Defensive: a future PreservationMode member with no arm here # would otherwise silently fall through. Treat as no-preserve. diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py index ea6d5897..c734f3d0 100644 --- a/src/coder_eval/reports.py +++ b/src/coder_eval/reports.py @@ -868,6 +868,10 @@ def _compute_suite_rollup( rows_passed = sum(1 for r in rows if r.result.final_status.category == "succeeded") rows_failed = sum(1 for r in rows if r.result.final_status.category == "failed") rows_error = sum(1 for r in rows if r.result.final_status.category == "error") + # An ungraded row was never measured, so it leaves BOTH sides of the rate — + # the same rule RunSummary.pass_rate and VariantAggregate.pass_rate follow. + rows_not_graded = sum(1 for r in rows if r.result.final_status.category == "ungraded") + rows_graded = rows_total - rows_not_graded scored = [r.result.weighted_score for r in rows if r.result.weighted_score is not None] average_weighted_score = sum(scored) / len(scored) if scored else None @@ -982,7 +986,8 @@ def _compute_suite_rollup( rows_passed=rows_passed, rows_failed=rows_failed, rows_error=rows_error, - pass_rate=rows_passed / rows_total if rows_total else 0.0, + rows_not_graded=rows_not_graded, + pass_rate=rows_passed / rows_graded if rows_graded else 0.0, average_weighted_score=average_weighted_score, criterion_stats=criterion_stats, failed_samples=failed_samples, @@ -998,8 +1003,9 @@ def _render_suite_markdown(rollup: SuiteRollup) -> str: "", f"**Variant**: `{rollup.variant_id}`", ( - f"**Rows**: {rollup.rows_total} total — " - f"{rollup.rows_passed} passed, {rollup.rows_failed} failed, {rollup.rows_error} errored" + f"**Rows**: {rollup.rows_total} total — {rollup.rows_passed} passed, " + + f"{rollup.rows_failed} failed, {rollup.rows_error} errored" + + (f", {rollup.rows_not_graded} not graded" if rollup.rows_not_graded else "") ), f"**Pass rate**: {rollup.pass_rate * 100:.1f}%", ] diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index e3cdd95f..ea1aceac 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -26,6 +26,7 @@ describe_prompt_config, fmt_mean_sd, fmt_p, + format_score, load_variant_eval_results, paired_comparison, stddev, @@ -248,7 +249,8 @@ def generate_task_report(summary: TaskExperimentSummary) -> str: tokens_str = f"{v.total_tokens:,}" if v.total_tokens is not None else "N/A" avg_dur = v.duration_seconds / v.replicate_count lines.append( - f"| {v.variant_id} | {v.weighted_score:.3f} | {v.final_status}" + f" | {avg_dur:.1f}s | {tokens_str} |" + f"| {v.variant_id} | {format_score(v.weighted_score)} | {v.final_status}" + + f" | {avg_dur:.1f}s | {tokens_str} |" ) return "\n".join(lines) @@ -483,7 +485,7 @@ def _win_loss_lines(result: ExperimentResult) -> list[str]: vr = scores_by_variant.get(vid) if vr: status_icon = vr.final_status.icon - cells.append(f"{vr.weighted_score:.3f} ({status_icon})") + cells.append(f"{format_score(vr.weighted_score)} ({status_icon})") else: cells.append("N/A") best_str = f"{'TIE' if ts.is_tie else ts.best_variant}" @@ -639,7 +641,7 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir: variant_results = [ vr for ts in result.task_summaries for vr in ts.variant_results if vr.variant_id == variant_id ] - scores = [vr.weighted_score for vr in variant_results] + scores = [vr.weighted_score for vr in variant_results if vr.weighted_score is not None] durations = [vr.duration_seconds / vr.replicate_count for vr in variant_results] if scores and len(scores) >= 2: @@ -673,7 +675,8 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir: for vr in ts.variant_results: if vr.variant_id == variant_id: avg_duration = vr.duration_seconds / vr.replicate_count - row = f"| {ts.task_id} | {vr.weighted_score:.3f} | {vr.final_status} | {avg_duration:.1f}s |" + score_text = format_score(vr.weighted_score) + row = f"| {ts.task_id} | {score_text} | {vr.final_status} | {avg_duration:.1f}s |" if has_reps: row += f" {vr.replicate_count} |" if has_similarity: diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py index 7f5d4468..128c24ed 100644 --- a/src/coder_eval/reports_html.py +++ b/src/coder_eval/reports_html.py @@ -20,6 +20,7 @@ from coder_eval.models import FinalStatus, eval_result_total_cost, sum_costs from .reports import early_stop_gate_note +from .reports_stats import format_score if TYPE_CHECKING: @@ -1124,7 +1125,7 @@ def _variant_stddev_lines(variant_id: str, result: ExperimentResult | None) -> s from .reports_stats import stddev vrs = [vr for ts in result.task_summaries for vr in ts.variant_results if vr.variant_id == variant_id] - scores = [vr.weighted_score for vr in vrs] + scores = [vr.weighted_score for vr in vrs if vr.weighted_score is not None] durations = [vr.duration_seconds for vr in vrs] extras: list[str] = [] if len(scores) >= 2: @@ -1451,7 +1452,7 @@ def _experiment_per_task_comparison(result: ExperimentResult) -> str: if vr is None: cells.append("N/A") else: - cells.append(f"{vr.weighted_score:.3f} ({_esc(vr.final_status.icon)})") + cells.append(f"{format_score(vr.weighted_score)} ({_esc(vr.final_status.icon)})") best = "TIE" if ts.is_tie else ts.best_variant cells.append(f"{_esc(best)}") cells.append(f"{ts.score_spread:.3f}") @@ -1581,6 +1582,15 @@ def generate_variant_html( ) stddev_lines = _variant_stddev_lines(variant_id, result) rich_sections = _variant_rich_sections(variant_id, result, run_dir) + # Only rendered when non-zero, so an ordinary graded run's tile is + # unchanged — but a `coder-eval execute` run says where its tasks went + # instead of showing Succeeded/Failed/Errors all at zero. + ungraded_stat = ( + '
Not Graded
' + + f'
{agg.tasks_not_graded}
' + if agg.tasks_not_graded > 0 + else "" + ) budget_stats = "" if agg.tasks_token_budget_exceeded > 0: budget_stats += ( @@ -1609,6 +1619,7 @@ def generate_variant_html(
Succeeded
{agg.tasks_succeeded}
Failed
{agg.tasks_failed}
Errors
{agg.tasks_error}
+ {ungraded_stat} {budget_stats} {stddev_lines} @@ -1650,7 +1661,7 @@ def generate_experiment_html( {_esc(vid)} {_score_pill(agg.average_score)} - {agg.tasks_succeeded}/{agg.tasks_run} + {agg.tasks_succeeded}/{agg.tasks_graded} """ ) diff --git a/src/coder_eval/reports_stats.py b/src/coder_eval/reports_stats.py index 25b8c1c4..c123764b 100644 --- a/src/coder_eval/reports_stats.py +++ b/src/coder_eval/reports_stats.py @@ -310,6 +310,17 @@ class VariantSeries(NamedTuple): asst_turns: list[float] +# What an ungraded row shows where a score would go. Deliberately not "0.000": +# an ungraded task was never measured, and a zero is indistinguishable from a +# task that was measured and scored nothing. +UNGRADED_SCORE_TEXT = "n/a" + + +def format_score(score: float | None) -> str: + """Render a weighted score for a report table, or ``n/a`` when ungraded.""" + return UNGRADED_SCORE_TEXT if score is None else f"{score:.3f}" + + def collect_variant_series(result: ExperimentResult) -> dict[str, VariantSeries]: """Per-variant (scores, durations, tokens, assistant-turns) series, keyed by variant id. @@ -324,6 +335,14 @@ def collect_variant_series(result: ExperimentResult) -> dict[str, VariantSeries] s = series.get(vr.variant_id) if s is None: # a task result for a variant not in variant_ids continue + if vr.weighted_score is None: + # Ungraded row: no score exists, and appending 0.0 would enter a + # fabricated data point into every statistic below. Skip the row + # WHOLE rather than just its score — paired_comparison pairs the + # series across variants by index, so dropping one field would + # misalign them. `grade` is run-level, so an experiment is either + # entirely graded or entirely ungraded; this never splits a pair. + continue s.scores.append(vr.weighted_score) s.durations.append(vr.duration_seconds / vr.replicate_count) if vr.total_tokens is not None: diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 3fbead1b..51734c69 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -163,6 +163,12 @@ def __init__( self.sandbox_dir: Path | None = None self.venv_dir: Path | None = None self._cleanup_on_exit = True + # True once adopt() takes over an existing workspace. Read by the + # Orchestrator to suppress every step that would MUTATE the tree it was + # asked to grade (pre_run/post_run above all — several in-tree tasks + # copy fixtures over the workspace there) and to keep sandbox_path in + # the result, since an adopted directory outlives cleanup(). + self.was_adopted = False self.installed_tool_versions: dict[str, str] = {} self._command_base_path: str | None = None # Cached canonical `node_modules/@uipath`; pins UiPath CLI plugin discovery @@ -318,6 +324,7 @@ def adopt(self, workspace: Path) -> Path: self.sandbox_dir = workspace.resolve() # Never flipped True: an adopted directory belongs to the caller. self._cleanup_on_exit = False + self.was_adopted = True # Only NON-materializing steps below. Deliberately skipped, and why: # _setup_template would overwrite the workspace being graded diff --git a/tests/test_cleanup_preservation_guard.py b/tests/test_cleanup_preservation_guard.py index e02c2f8e..b2519cd8 100644 --- a/tests/test_cleanup_preservation_guard.py +++ b/tests/test_cleanup_preservation_guard.py @@ -201,6 +201,10 @@ async def test_none_without_workspace_dir_discards_path(tmp_path) -> None: orchestrator.workspace_dir = None orchestrator.result.sandbox_path = "/stale/path" # must be cleared mock_sandbox = MagicMock() + # A real Sandbox that was not adopted. Explicit because a bare MagicMock + # attribute is truthy, which would take the adopted arm (that one KEEPS the + # path, since an adopted directory survives cleanup) and hide the discard. + mock_sandbox.was_adopted = False orchestrator.sandbox = mock_sandbox await orchestrator._cleanup() diff --git a/tests/test_cli_telemetry.py b/tests/test_cli_telemetry.py index 1c757524..06f49cad 100644 --- a/tests/test_cli_telemetry.py +++ b/tests/test_cli_telemetry.py @@ -79,7 +79,7 @@ def test_help_never_crashes_when_telemetry_enabled_and_config_unwritable(tmp_pat async def test_run_emits_run_start_and_flushes(tmp_path): - summary = Mock(tasks_failed=0, tasks_error=0) + summary = Mock(tasks_failed=0, tasks_error=0, tasks_not_graded=0) with ( patch("coder_eval.cli.run_command.prepare_run_directory", return_value=tmp_path), @@ -115,7 +115,7 @@ async def test_run_emits_run_start_and_flushes(tmp_path): async def test_run_start_uses_default_fallbacks_for_none_inputs(tmp_path): # agent_type=None / stream_mode=None must surface as the "default"/"none" # fallback property values, not as null. - summary = Mock(tasks_failed=0, tasks_error=0) + summary = Mock(tasks_failed=0, tasks_error=0, tasks_not_graded=0) with ( patch("coder_eval.cli.run_command.prepare_run_directory", return_value=tmp_path), diff --git a/tests/test_execute_evaluate_loop.py b/tests/test_execute_evaluate_loop.py index 9ab17528..96c0b415 100644 --- a/tests/test_execute_evaluate_loop.py +++ b/tests/test_execute_evaluate_loop.py @@ -11,6 +11,7 @@ from __future__ import annotations import json +import shutil from pathlib import Path from typing import Any @@ -204,6 +205,54 @@ def test_execute_resume_treats_an_executed_row_as_done(tmp_path: Path) -> None: assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.NOT_GRADED.value +def test_a_detached_grade_does_not_re_run_pre_run_against_the_workspace(tmp_path: Path) -> None: + """`run()` calls the pre/post-run hooks unconditionally, with cwd = the + sandbox. On an ADOPTED sandbox that sandbox is the agent's own output, and + several in-tree tasks stage fixtures there (`cp -a /app/[!.]* "$PWD/"`), so + re-running them would overwrite the deliverables before the criteria read + them — changing the verdict and destroying preserved artifacts.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + proof = sorted(run_dir.glob("**/artifacts/**/proof.txt"))[0] + # Mark the agent's file. The fixture's pre_run rewrites proof.txt from + # scratch, so a re-run would wipe this marker. + proof.write_text("coder-eval-ran-without-a-coder AND-THE-AGENT-EDITED-THIS", encoding="utf-8") + + _invoke(["evaluate", str(task_dir)]) + + assert "AND-THE-AGENT-EDITED-THIS" in proof.read_text(encoding="utf-8"), ( + "pre_run re-ran against the adopted workspace and overwrote the agent's work" + ) + # The hooks' recorded outcomes are carried over rather than lost. + assert _row(task_dir)["pre_run_results"], "the execute phase's pre_run results were dropped" + + +def test_run_resume_exits_non_zero_when_it_cannot_grade(tmp_path: Path) -> None: + """`run` was asked for a verdict. If grading fails, reporting exit 0 tells CI + the suite is fine when nothing was actually scored.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + # Remove the workspace so the re-grade has nothing to grade against. + shutil.rmtree(run_dir / "default" / "agentless_smoke_test" / "00" / "artifacts", ignore_errors=True) + row = _task_dir(run_dir) / "task.json" + record = json.loads(row.read_text(encoding="utf-8")) + record["sandbox_path"] = str(tmp_path / "gone") + row.write_text(json.dumps(record), encoding="utf-8") + + result = runner.invoke(app, ["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert result.exit_code != 0, "a run that graded nothing must not report success" + + +def test_execute_still_exits_zero_with_every_row_ungraded(tmp_path: Path) -> None: + """The other side of the rule above: under `execute` an ungraded row is the + expected outcome, not a failure of the command.""" + run_dir = tmp_path / "r" + result = runner.invoke(app, ["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + assert result.exit_code == 0, result.output + + def test_execute_to_run_resume_emits_no_config_drift_warning(tmp_path: Path) -> None: """`grade` is exempt from the fingerprint diff: this flow is supported, and the warning's "keeps their original-config results" text is wrong for it.""" diff --git a/tests/test_post_run.py b/tests/test_post_run.py index a55f0e14..a5b5687f 100644 --- a/tests/test_post_run.py +++ b/tests/test_post_run.py @@ -119,7 +119,7 @@ def _make_orchestrator(task: TaskDefinition, tmp_path: Path) -> Orchestrator: async def test_post_run_skipped_when_empty(tmp_path): task = _make_task(post_run=[]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -142,7 +142,7 @@ async def test_post_run_skipped_when_no_sandbox(tmp_path): async def test_post_run_command_success(tmp_path): task = _make_task(post_run=[PostRunCommand(command="echo '{\"ok\": true}'")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -161,7 +161,7 @@ async def test_post_run_command_failure_does_not_affect_result(tmp_path): ) orch = _make_orchestrator(task, tmp_path) orch.result.final_status = FinalStatus.SUCCESS - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -178,7 +178,7 @@ async def test_post_run_command_with_pipes(tmp_path): """Shell commands support pipes and redirects.""" task = _make_task(post_run=[PostRunCommand(command="echo hello world | tr a-z A-Z")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -192,7 +192,7 @@ async def test_post_run_command_with_pipes(tmp_path): async def test_post_run_command_timeout(tmp_path): task = _make_task(post_run=[PostRunCommand(command="sleep 10", timeout=1)]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -211,7 +211,7 @@ async def test_post_run_multiple_commands(tmp_path): ] ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -229,7 +229,7 @@ async def test_post_run_cwd_is_sandbox(tmp_path): task = _make_task(post_run=[PostRunCommand(command='python3 -c "import os; print(os.getcwd())"')]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = sandbox_dir await orch._run_post_run_commands() @@ -243,7 +243,7 @@ async def test_post_run_streams_stdout_to_logger(tmp_path, caplog): """Each line of stdout is forwarded to the orchestrator logger as it is read.""" task = _make_task(post_run=[PostRunCommand(command="python3 -c \"print('line-one'); print('line-two')\"")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with caplog.at_level(logging.INFO, logger="coder_eval.orchestrator"): @@ -263,7 +263,7 @@ async def test_post_run_streams_stderr_as_warning(tmp_path, caplog): """Stderr lines are forwarded at WARNING level (separate from stdout).""" task = _make_task(post_run=[PostRunCommand(command="python3 -c \"import sys; print('boom', file=sys.stderr)\"")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with caplog.at_level(logging.WARNING, logger="coder_eval.orchestrator"): @@ -279,7 +279,7 @@ async def test_post_run_output_truncated(tmp_path): # Generate output larger than the limit task = _make_task(post_run=[PostRunCommand(command="python3 -c \"print('x' * 200_000)\"")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() diff --git a/tests/test_pre_run.py b/tests/test_pre_run.py index 6febaaa7..947039ed 100644 --- a/tests/test_pre_run.py +++ b/tests/test_pre_run.py @@ -150,7 +150,7 @@ def _make_orchestrator(task: TaskDefinition, tmp_path: Path) -> Orchestrator: async def test_pre_run_skipped_when_empty(tmp_path): task = _make_task(pre_run=[]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -173,7 +173,7 @@ async def test_pre_run_skipped_when_no_sandbox(tmp_path): async def test_pre_run_command_success(tmp_path): task = _make_task(pre_run=[PreRunCommand(command="echo '{\"ok\": true}'")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -189,7 +189,7 @@ async def test_pre_run_command_success(tmp_path): async def test_pre_run_command_with_pipes(tmp_path): task = _make_task(pre_run=[PreRunCommand(command="echo hello world | tr a-z A-Z")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -208,7 +208,7 @@ async def test_pre_run_multiple_commands(tmp_path): ] ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -225,7 +225,7 @@ async def test_pre_run_cwd_is_sandbox(tmp_path): task = _make_task(pre_run=[PreRunCommand(command='python3 -c "import os; print(os.getcwd())"')]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = sandbox_dir await orch._run_pre_run_commands() @@ -238,7 +238,7 @@ async def test_pre_run_cwd_is_sandbox(tmp_path): async def test_pre_run_streams_stdout_to_logger(tmp_path, caplog): task = _make_task(pre_run=[PreRunCommand(command="python3 -c \"print('line-one'); print('line-two')\"")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with caplog.at_level(logging.INFO, logger="coder_eval.orchestrator"): @@ -263,7 +263,7 @@ async def test_pre_run_streams_stderr_as_warning(tmp_path, caplog): ] ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with caplog.at_level(logging.WARNING, logger="coder_eval.orchestrator"): @@ -277,7 +277,7 @@ async def test_pre_run_streams_stderr_as_warning(tmp_path, caplog): async def test_pre_run_output_truncated(tmp_path): task = _make_task(pre_run=[PreRunCommand(command="python3 -c \"print('x' * 200_000)\"")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -296,7 +296,7 @@ async def test_pre_run_failure_raises_when_fail_on_error_true(tmp_path): pre_run=[PreRunCommand(command='python3 -c "import sys; sys.exit(1)"')], ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with pytest.raises(RuntimeError, match="Pre-run command failed"): @@ -307,7 +307,7 @@ async def test_pre_run_failure_raises_when_fail_on_error_true(tmp_path): async def test_pre_run_timeout_raises_when_fail_on_error_true(tmp_path): task = _make_task(pre_run=[PreRunCommand(command="sleep 10", timeout=1)]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with pytest.raises(RuntimeError, match="timed out after 1s"): @@ -320,7 +320,7 @@ async def test_pre_run_failure_result_captured_before_raise(tmp_path): pre_run=[PreRunCommand(command='python3 -c "import sys; sys.exit(2)"')], ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with pytest.raises(RuntimeError): @@ -339,7 +339,7 @@ async def test_pre_run_subsequent_commands_skipped_after_abort(tmp_path): ], ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with pytest.raises(RuntimeError): @@ -362,7 +362,7 @@ async def test_pre_run_failure_does_not_raise_when_fail_on_error_false(tmp_path) ], ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -376,7 +376,7 @@ async def test_pre_run_spawn_exception_raises_when_fail_on_error_true(tmp_path): """Generic exceptions from create_subprocess_shell propagate as RuntimeError when fail_on_error=True.""" task = _make_task(pre_run=[PreRunCommand(command="echo hi")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with ( @@ -399,7 +399,7 @@ async def test_pre_run_spawn_exception_does_not_raise_when_fail_on_error_false(t ] ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path real_shell = __import__("asyncio").create_subprocess_shell @@ -428,7 +428,7 @@ async def test_pre_run_timeout_does_not_raise_when_fail_on_error_false(tmp_path) ], ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() diff --git a/tests/test_regrade.py b/tests/test_regrade.py new file mode 100644 index 00000000..668046c8 --- /dev/null +++ b/tests/test_regrade.py @@ -0,0 +1,259 @@ +"""``orchestration/regrade.py`` — the refusals, not the happy path. + +The end-to-end loop test covers a successful re-grade of the agentless task. What +it cannot cover is every branch that REFUSES to grade, and those are the ones that +matter: each exists because grading anyway would publish a plausible number that +is wrong. The reference-digest guard in particular shipped as dead code (nothing +wrote the key it read) precisely because the only test that reached it used a +fixture with no reference at all. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path + +import pytest + +from coder_eval.models import ( + AgentKind, + EvaluationResult, + FileExistsCriterion, + FinalStatus, + RunCommandCriterion, + TaskConfigRecord, + TaskDefinition, + parse_agent_config, +) +from coder_eval.orchestration.regrade import ( + PRE_GRADE_JSON, + TASK_JSON, + RegradeError, + back_up_pre_grade_record, + default_workspace, + load_prior_result, + task_from_prior, + verify_reference_unchanged, +) +from coder_eval.path_utils import digest_tree + + +def _task(*, reference: dict[str, str] | None = None, command: str | None = None) -> TaskDefinition: + criteria: list[object] = [FileExistsCriterion(path="x.txt", description="x")] + if command is not None: + criteria.append(RunCommandCriterion(command=command, description="run it")) + return TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + reference=reference, # type: ignore[arg-type] + success_criteria=criteria, # type: ignore[arg-type] + ) + + +def _result(**kwargs: object) -> EvaluationResult: + from datetime import datetime + + base: dict[str, object] = { + "task_id": "t", + "task_description": "d", + "variant_id": "v", + "agent_type": AgentKind.CLAUDE_CODE, + "started_at": datetime(2020, 1, 1), + "final_status": FinalStatus.NOT_GRADED, + "iteration_count": 1, + } + base.update(kwargs) + return EvaluationResult(**base) # type: ignore[arg-type] + + +# -------------------------------------------------------------------------- +# load_prior_result +# -------------------------------------------------------------------------- + + +def test_missing_task_json_is_a_regrade_error(tmp_path: Path) -> None: + with pytest.raises(RegradeError, match="Cannot read"): + load_prior_result(tmp_path) + + +def test_unparseable_task_json_is_a_regrade_error(tmp_path: Path) -> None: + (tmp_path / TASK_JSON).write_text("{not json", encoding="utf-8") + with pytest.raises(RegradeError, match="not a readable EvaluationResult"): + load_prior_result(tmp_path) + + +# -------------------------------------------------------------------------- +# task_from_prior — which task gets graded +# -------------------------------------------------------------------------- + + +def test_no_task_config_refuses_rather_than_guessing(tmp_path: Path) -> None: + with pytest.raises(RegradeError, match="carries no task_config"): + task_from_prior(_result(), tmp_path) + + +def test_resolved_config_wins_over_the_source_yaml(tmp_path: Path) -> None: + """`resolved` is post-merge, so it carries variant overrides / -D / dataset + expansion. Re-reading the YAML would grade a DIFFERENT task.""" + source = tmp_path / "t.yaml" + source.write_text("task_id: from-yaml\n", encoding="utf-8") + resolved = _task().model_dump(mode="json") + resolved["task_id"] = "from-resolved" + prior = _result(task_config=TaskConfigRecord(resolved=resolved, source_yaml="raw", source_file=str(source))) + + task, _ = task_from_prior(prior, tmp_path) + + assert task.task_id == "from-resolved" + + +def test_unusable_resolved_config_with_no_source_refuses(tmp_path: Path) -> None: + prior = _result(task_config=TaskConfigRecord(resolved={"nonsense": True}, source_yaml="raw", source_file=None)) + with pytest.raises(RegradeError, match="no longer validates"): + task_from_prior(prior, tmp_path) + + +def test_source_fallback_is_loud(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + """A quiet fallback would silently grade a task other than the one that ran.""" + source = tmp_path / "t.yaml" + source.write_text( + "task_id: from-yaml\ndescription: d\ninitial_prompt: p\n" + + "success_criteria:\n - type: file_exists\n path: x.txt\n description: x\n", + encoding="utf-8", + ) + prior = _result( + task_config=TaskConfigRecord(resolved={"nonsense": True}, source_yaml="raw", source_file=str(source)) + ) + + with caplog.at_level(logging.WARNING): + task, _ = task_from_prior(prior, tmp_path) + + assert task.task_id == "from-yaml" + assert "NOT reapplied" in caplog.text + + +def test_shell_commands_from_a_run_dir_config_are_announced(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + """A run dir is a shareable artifact, and rebuilding from it decides what the + grader executes. Intended, but never silent.""" + resolved = _task(command="echo surprising").model_dump(mode="json") + prior = _result(task_config=TaskConfigRecord(resolved=resolved, source_yaml="raw", source_file=None)) + + with caplog.at_level(logging.WARNING): + task_from_prior(prior, tmp_path) + + assert "echo surprising" in caplog.text + + +# -------------------------------------------------------------------------- +# default_workspace +# -------------------------------------------------------------------------- + + +def test_recorded_sandbox_path_wins_when_it_still_exists(tmp_path: Path) -> None: + workspace = tmp_path / "ws" + workspace.mkdir() + assert default_workspace(tmp_path, _result(sandbox_path=str(workspace))) == workspace + + +def test_falls_back_to_the_single_artifacts_child(tmp_path: Path) -> None: + child = tmp_path / "artifacts" / "t" + child.mkdir(parents=True) + assert default_workspace(tmp_path, _result(sandbox_path="/gone")) == child + + +def test_flat_artifacts_dir_is_itself_the_workspace(tmp_path: Path) -> None: + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + (artifacts / "file.txt").write_text("x", encoding="utf-8") + assert default_workspace(tmp_path, _result()) == artifacts + + +def test_no_workspace_at_all_refuses(tmp_path: Path) -> None: + with pytest.raises(RegradeError, match="No workspace to grade"): + default_workspace(tmp_path, _result()) + + +# -------------------------------------------------------------------------- +# verify_reference_unchanged — the anti-cheat guard +# -------------------------------------------------------------------------- + + +def _reference_task(tmp_path: Path) -> tuple[TaskDefinition, Path, Path]: + task_file = tmp_path / "t.yaml" + task_file.write_text("x", encoding="utf-8") + reference = tmp_path / "ref" + reference.mkdir() + (reference / "answer.py").write_text("print('right')\n", encoding="utf-8") + return _task(reference={"directory": "ref"}), task_file, reference + + +def test_an_edited_reference_refuses_the_grade(tmp_path: Path) -> None: + """The headline guarantee. Without it, an answer key edited between execute + and grade scores the agent's old work against a new one.""" + task, task_file, reference = _reference_task(tmp_path) + prior = _result(environment_info={"reference_digest": digest_tree(reference)}) + verify_reference_unchanged(prior, task, task_file) # unchanged: fine + + (reference / "answer.py").write_text("print('different')\n", encoding="utf-8") + + with pytest.raises(RegradeError, match="digest mismatch"): + verify_reference_unchanged(prior, task, task_file) + + +def test_a_vanished_reference_refuses_rather_than_grading_without_one(tmp_path: Path) -> None: + task, task_file, reference = _reference_task(tmp_path) + prior = _result(environment_info={"reference_digest": digest_tree(reference)}) + for p in reference.iterdir(): + p.unlink() + reference.rmdir() + + with pytest.raises(RegradeError): + verify_reference_unchanged(prior, task, task_file) + + +def test_a_run_without_a_recorded_digest_says_so(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + """Silence here is what let the guard ship as dead code for a release.""" + task, task_file, _ = _reference_task(tmp_path) + + with caplog.at_level(logging.WARNING): + verify_reference_unchanged(_result(), task, task_file) + + assert "cannot be verified" in caplog.text + + +def test_a_task_with_no_reference_is_not_checked(tmp_path: Path) -> None: + verify_reference_unchanged(_result(), _task(), tmp_path / "t.yaml") + + +# -------------------------------------------------------------------------- +# back_up_pre_grade_record +# -------------------------------------------------------------------------- + + +def test_the_pre_grade_record_is_written_once(tmp_path: Path) -> None: + """A second grade must not overwrite the ORIGINAL execute record with an + already-graded one — that is the only evidence the run was ungraded.""" + (tmp_path / TASK_JSON).write_text('{"round": 1}', encoding="utf-8") + back_up_pre_grade_record(tmp_path) + (tmp_path / TASK_JSON).write_text('{"round": 2}', encoding="utf-8") + back_up_pre_grade_record(tmp_path) + + assert json.loads((tmp_path / PRE_GRADE_JSON).read_text(encoding="utf-8")) == {"round": 1} + + +def test_backup_is_a_no_op_with_nothing_to_back_up(tmp_path: Path) -> None: + back_up_pre_grade_record(tmp_path) + assert not (tmp_path / PRE_GRADE_JSON).exists() + + +def test_a_failed_backup_never_fails_the_grade(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The audit copy is a convenience; the verdict is the deliverable.""" + (tmp_path / TASK_JSON).write_text("{}", encoding="utf-8") + + def _boom(*_args: object, **_kwargs: object) -> None: + raise OSError("read-only file system") + + monkeypatch.setattr(Path, "write_text", _boom) + back_up_pre_grade_record(tmp_path) # must not raise diff --git a/tests/test_run_command_junit.py b/tests/test_run_command_junit.py index b5ac1de0..7f8e5341 100644 --- a/tests/test_run_command_junit.py +++ b/tests/test_run_command_junit.py @@ -26,7 +26,7 @@ async def _invoke( status: str, failed: bool, ) -> None: - summary = Mock(tasks_failed=1 if failed else 0, tasks_error=0) + summary = Mock(tasks_failed=1 if failed else 0, tasks_error=0, tasks_not_graded=0) async def _fake(*_args, **_kwargs): # Mirror production: run.json is persisted inside _run_with_experiment. diff --git a/tests/test_run_metrics.py b/tests/test_run_metrics.py index 6ac5baff..6866a87f 100644 --- a/tests/test_run_metrics.py +++ b/tests/test_run_metrics.py @@ -5,7 +5,9 @@ **The denominator.** ``pass_rate`` used to be ``succeeded / (run - error)``, which paid a bonus for erroring: the more a run fell over, the smaller its denominator got, up to the degenerate case of a run rendering as a perfect score while passing -a handful of rows. Every surface now divides by ``tasks_run``. +a handful of rows. Every surface now divides by ``tasks_graded`` — every +dispatched task except the ones that were never measured at all (``coder-eval +execute`` leaves rows ``NOT_GRADED``, and those leave BOTH sides of the rate). **The bill.** Cost was summed over whatever rows happened to carry one, so a run whose model was missing from the rate card, or whose turns were killed before the diff --git a/tests/test_seed_from_prior_result.py b/tests/test_seed_from_prior_result.py new file mode 100644 index 00000000..319815d3 --- /dev/null +++ b/tests/test_seed_from_prior_result.py @@ -0,0 +1,272 @@ +"""``Orchestrator._seed_from_prior_result`` — the detached grade's fidelity contract. + +A detached grade (``evaluate `` / ``run --resume``) recomputes the +verdict but must not recompute the RUN. Every field it carries over is a fact the +agent phase established and this pass cannot re-derive; every field it does not +carry is either recomputed from the trajectory or deliberately dropped. + +The end-to-end tests in ``test_execute_evaluate_loop.py`` exercise this through a +fixture where most of these fields hold their defaults, so deleting a carry line +leaves them green. These tests set every field to a distinctive value instead, and +the partition below fails closed when a new field is added to ``EvaluationResult`` +without a decision about it. +""" + +from __future__ import annotations + +from datetime import datetime +from pathlib import Path + +import pytest + +from coder_eval.models import ( + AgentKind, + CommandExecutedCriterion, + CriterionResult, + EarlyStopInfo, + EarlyStopReason, + EvaluationResult, + FileExistsCriterion, + FinalStatus, + PostRunResult, + TaskDefinition, + parse_agent_config, +) +from coder_eval.orchestrator import Orchestrator + + +# Every field on EvaluationResult, partitioned by what a detached grade does with +# it. No catch-all: a new field fails the parity test below until it is listed, +# which is the same fail-closed shape as _STATUS_CATEGORIES in models/enums.py. +CARRIED = { + "started_at", + "iterations", + "iteration_count", + "early_stop", + "max_turns_exhausted", + "error_message", + "error_details", + "error_log_tail", + "sdk_options", + "agent_config", + "expected_commands", + "simulation", + "pre_run_results", + "post_run_results", + "sandbox_path", + "environment_info", +} + +# Recomputed by this pass — carrying them would defeat the point. +RECOMPUTED = { + # The verdict itself: exactly what the grading pass produces. + "final_status", + "weighted_score", + "success_criteria_results", + "post_failure_criteria_results", + # Derived from `iterations`, which IS carried — so seeding the trajectory + # reproduces these exactly without copying them. + "model_used", + "command_stats", + "total_token_usage", + "total_assistant_turns", + "actual_commands", + "commands_efficiency", + # Identity, supplied by the caller from the task being graded. + "task_id", + "task_description", + "variant_id", + "agent_type", + "task_config", + # Timing: `duration_seconds` is restored from the prior result in + # _finalize_result (after its own timing write), and `completed_at` marks + # when the row reached its final state, which the grade genuinely changes. + "duration_seconds", + "completed_at", +} + + +def _prior() -> EvaluationResult: + """A prior result with a distinctive value in every carried field.""" + return EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2020, 1, 1, 0, 0, 0), + final_status=FinalStatus.NOT_GRADED, + iteration_count=7, + max_turns_exhausted=True, + error_message="prior message", + error_details={"where": "prior"}, + error_log_tail="prior tail", + sdk_options={"opt": "prior"}, + agent_config=parse_agent_config(type=AgentKind.CLAUDE_CODE, model="prior-model"), + expected_commands=11, + pre_run_results=[PostRunResult(command="prior-pre", exit_code=0)], + post_run_results=[PostRunResult(command="prior-post", exit_code=0)], + sandbox_path="/prior/workspace", + environment_info={"installed_tools": "prior"}, + early_stop=EarlyStopInfo( + reason=EarlyStopReason.CRITERION_FAILED, + deciding_criterion_type="skill_triggered", + deciding_criterion_description="the armed criterion", + sdk_turn_index=0, + tool_call_index=1, + elapsed_seconds=1.0, + gate_threshold=1.0, + ), + ) + + +def _seeded(tmp_path: Path) -> tuple[Orchestrator, EvaluationResult]: + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + prior = _prior() + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v", prior_result=prior) + orch.result = EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2030, 1, 1, 0, 0, 0), + final_status=FinalStatus.FAILURE, + iteration_count=0, + environment_info={"installed_tools": "grader"}, + ) + orch._seed_from_prior_result() + assert orch.result is not None + return orch, prior + + +def test_field_partition_covers_every_evaluation_result_field() -> None: + """The sensor. A field added to EvaluationResult must be classified as + carried or recomputed before this passes — otherwise it silently defaults on + every detached grade, which is exactly how `agent_config` was being lost.""" + classified = CARRIED | RECOMPUTED + fields = set(EvaluationResult.model_fields) + assert not fields - classified, ( + f"Unclassified EvaluationResult field(s): {sorted(fields - classified)}. Decide whether " + "_seed_from_prior_result must carry them, then add them to CARRIED or RECOMPUTED." + ) + assert not classified - fields, f"Stale entries: {sorted(classified - fields)}" + + +@pytest.mark.parametrize("field", sorted(CARRIED - {"environment_info"})) +def test_every_carried_field_reaches_the_regrade(field: str, tmp_path: Path) -> None: + orch, prior = _seeded(tmp_path) + assert orch.result is not None + assert getattr(orch.result, field) == getattr(prior, field), ( + f"_seed_from_prior_result dropped `{field}`; the graded row would report its default " + "instead of what the run actually did." + ) + + +def test_early_stop_is_carried_because_it_selects_the_gate(tmp_path: Path) -> None: + """Called out separately because it is the one carried field that changes the + VERDICT: gate selection is FIRED-ONLY, so a dropped early_stop re-grades a + truncated trajectory under the full-run strict-AND gate.""" + orch, _ = _seeded(tmp_path) + assert orch.result is not None and orch.result.early_stop is not None + assert orch.result.early_stop.reason is EarlyStopReason.CRITERION_FAILED + + +def test_grader_environment_is_kept_beside_the_run_s_not_over_it(tmp_path: Path) -> None: + orch, _ = _seeded(tmp_path) + assert orch.result is not None + # The run's own capture wins: a report showing the grader's tool versions as + # the run's is worse than one showing neither. + assert orch.result.environment_info["installed_tools"] == "prior" + assert orch.result.environment_info["graded_by"] == {"installed_tools": "grader"} + + +def test_the_evaluate_only_path_selects_the_same_gate_as_the_agent_path(tmp_path: Path) -> None: + """C1: gate selection is FIRED-ONLY, and a detached grade reaches the verdict + through the evaluate-only branch. That branch used to call + ``all_criteria_passed`` unconditionally, so re-grading an early-stopped run + applied the full-run strict-AND gate to a truncated trajectory and could flip + SUCCESS into FAILURE. Both paths must go through ``_select_gate``.""" + import inspect + + source = inspect.getsource(Orchestrator._evaluation_loop) + assert source.count("_select_gate()") == 2, ( + "Both the evaluate-only branch and the agent branch must select the gate through " + "_select_gate(); a second hand-written selection is how the seeded early_stop " + "came to be carried but never read." + ) + assert "all_criteria_passed" not in source, "gate selection belongs in _select_gate, not inline" + + # A truncated run: the ARMED criterion passed, the unarmed one never had the + # chance to. The armed gate says SUCCESS; strict-AND says FAILURE. That + # difference IS the flipped verdict, and it is decided purely by early_stop. + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[ + CommandExecutedCriterion( + description="armed", + tool_name="Read", + require_success=True, + stop_early={"on_pass": "stop"}, # type: ignore[arg-type] + ), + FileExistsCriterion(path="x.txt", description="unarmed"), + ], + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v", prior_result=_prior()) + orch.result = _prior() + orch.result.success_criteria_results = [ + CriterionResult(criterion_type="command_executed", description="armed", score=1.0), + CriterionResult(criterion_type="file_exists", description="unarmed", score=0.0), + ] + + assert orch._select_gate() is True, "an early-stopped run gates on the armed subset" + orch.result.early_stop = None + assert orch._select_gate() is False, "a run that completed naturally gates strict-AND" + + +def test_grading_cannot_overturn_an_execution_fact() -> None: + """H5: a detached grade re-runs the criteria over a trajectory it did not + produce. It may move NOT_GRADED to a verdict; it must not turn a run that + timed out or crashed into a pass.""" + assert not FinalStatus.NOT_GRADED.is_execution_fact + assert not FinalStatus.SUCCESS.is_execution_fact + assert not FinalStatus.FAILURE.is_execution_fact + for status in ( + FinalStatus.ERROR, + FinalStatus.TIMEOUT, + FinalStatus.BUILD_FAILED, + FinalStatus.MAX_TURNS_EXHAUSTED, + FinalStatus.TOKEN_BUDGET_EXCEEDED, + FinalStatus.COST_BUDGET_EXCEEDED, + ): + assert status.is_execution_fact, f"{status} describes the run, so grading must preserve it" + + +def test_seeding_is_a_no_op_without_a_prior_result(tmp_path: Path) -> None: + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v") + orch.result = EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2030, 1, 1), + final_status=FinalStatus.FAILURE, + iteration_count=0, + ) + orch._seed_from_prior_result() + assert orch.result.started_at == datetime(2030, 1, 1) + assert orch.result.iterations == [] From 489383d05d0a31c964021c644f141c5dbe18c07b Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Thu, 3 Sep 2026 15:41:24 -0700 Subject: [PATCH 05/11] fix(eval): address the medium and low findings from the branch review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 23 medium / 20 low findings from the same review pass. Grouped by what they change rather than by axis. Correctness * `Sandbox.adopt` discovered `/.venv` unconditionally, while `setup` only ever populates `venv_dir` when `config.python` is set. A venv the task never asked for was prepended to PATH and exported as VIRTUAL_ENV for every criterion — the exact divergence the `command_base_path` round trip exists to close, and a way for an agent to shadow binaries from its own workspace. * `default_workspace` inferred the workspace as "the single child of artifacts/". A dataset row's `task_id` is `/`, so that resolves one level too high and every path-relative criterion then fails as a locating artifact rather than as a verdict. It now resolves `artifacts/` exactly, and RAISES when ambiguous instead of guessing the parent. * `_write_back` overwrote the canonical `task.json` with a plain `write_text` while the orchestrator writes the same file via tmp + `os.replace`. A torn write parses as malformed, which `--resume` reads as "not complete" and pays for the agent again. One `write_text_atomic` helper now serves both. * A grading crash wrote `ERROR` over a re-gradeable `NOT_GRADED` row — and `ERROR` is "complete" for both commands, so the row could never be graded again. Both detached paths now keep the ungraded row. * `load_prior_result` sat outside the resume loop's `try`, so one unreadable row aborted the whole resume BEFORE `run_batch` — none of the `to_run` tasks executed either, the opposite of the documented "one bad row never aborts". * `back_up_pre_grade_record` ran after the orchestrator, so with `--run-dir` pointing at the target it captured an already-graded record — destroying the evidence it exists to preserve. It is now taken during input resolution. * `verify_reference_unchanged` moved INSIDE `regrade_in_place`: a guard a caller has to remember is one a third caller will forget. * `completed_at` is carried from the prior run, so a re-graded row's three time fields agree with each other. * `grade` is now coerced at the container boundary rather than annotated — `"false"` is a truthy str. Reporting * `VariantAggregate.average_score` is `float | None`; `_mean_graded_score` returned 0.0 for the case that actually happens (nothing graded), printing `Average Score: 0.000` beside `Pass Rate: n/a`. * `SuiteRollup` gains `rows_not_graded`, the graded denominator, and the row-count invariant its two siblings have and it did not. * `_seed_from_prior_result` nested a whole env capture under `graded_by`; `environment_info` is consumed as a FLAT map (the HTML report `_esc`apes each value into a cell), so it renders as a Python dict repr. Flattened to `graded_by_*` scalars, kept only where they differ, and a second grade no longer clobbers the first grader's stamp. * `command_base_path` is a full PATH string written on every run; it and the provenance keys are excluded from the rendered Environment tables. * The end-of-run hint pointed at `evaluate ` — the shape with NO trajectory, which scores trajectory-reading criteria differently from what `run` would have produced. An empty run also printed no Results line. Two new lint rules, each of which found a live instance the moment it ran * CE047 — an `environment_info` key that is read must be written somewhere in `src/`. This is the durable form of the `reference_digest` fix: the bag is `dict[str, Any]`, so nothing connects a reader to its writer, and a reader with no writer is silently inert. * CE048 — never call a Typer command function in process. It scans `tests/` as well, because that is the only place the defect occurs, and it immediately found six live calls to `plan_command` — whose body already carried an `isinstance(experiment, Path)` guard papering the sentinel over. Split into `run_plan`, matching `run_pipeline` / `run_evaluation`. Also: `TASK_JSON` / `.venv` are single constants in `path_utils` instead of two half-copies plus ten literals; symlink refusal and a containment check on the paths a shared run dir supplies; `evaluate --help`'s usage line no longer renders `[]`; `--resume` and `--preserve` help match the behavior; the resumable-dataset constraint, `task.execute.json` and the suite schema are documented. Tests: `test_ungraded_reporting.py` (JUnit ``, the switched Markdown denominator, the console summary, and the `VariantAggregate` twin of the four `RunSummary` cases), `test_detached_grading_guards.py` (the simulation refusal, `--in-place`/`--copy` selection, the PATH round trip and its filter, the LiteLLM skip), plus grade-idempotence, `--workspace`, the execution-fact refusal, the resume error paths and a `/`-bearing dataset id. make verify green (4688 passed, 92.26%); evalboard 621 tests green. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/shared/run-layout.md | 1 + CLAUDE.md | 2 +- docs/REPORT_SCHEMA.md | 7 +- docs/USER_GUIDE.md | 10 +- plugins/coder-eval/reference/run-layout.md | 1 + src/coder_eval/cli/evaluate_command.py | 60 +++- src/coder_eval/cli/evaluate_target.py | 9 +- src/coder_eval/cli/plan_command.py | 15 +- src/coder_eval/cli/run_command.py | 55 +++- src/coder_eval/cli/run_helpers.py | 13 +- .../cli/run_task_internal_command.py | 10 +- src/coder_eval/models/experiment.py | 5 +- src/coder_eval/orchestration/experiment.py | 15 +- src/coder_eval/orchestration/regrade.py | 97 +++++-- src/coder_eval/orchestrator.py | 41 ++- src/coder_eval/path_utils.py | 27 ++ src/coder_eval/reports_experiment.py | 5 +- src/coder_eval/reports_html.py | 6 +- src/coder_eval/reports_stats.py | 13 + src/coder_eval/sandbox.py | 22 +- .../rules/ce047_env_info_key_round_trip.py | 122 ++++++++ .../ce048_no_in_process_typer_command_call.py | 106 +++++++ tests/lint/runner.py | 4 + tests/test_custom_lint.py | 10 +- tests/test_detached_grading_guards.py | 261 ++++++++++++++++++ tests/test_early_stop.py | 6 +- tests/test_execute_command.py | 8 +- tests/test_execute_evaluate_loop.py | 96 +++++++ tests/test_plan_command.py | 22 +- tests/test_regrade.py | 55 +++- tests/test_seed_from_prior_result.py | 12 +- tests/test_ungraded_reporting.py | 254 +++++++++++++++++ 32 files changed, 1258 insertions(+), 112 deletions(-) create mode 100644 tests/lint/rules/ce047_env_info_key_round_trip.py create mode 100644 tests/lint/rules/ce048_no_in_process_typer_command_call.py create mode 100644 tests/test_detached_grading_guards.py create mode 100644 tests/test_ungraded_reporting.py diff --git a/.claude/shared/run-layout.md b/.claude/shared/run-layout.md index 57edc4a6..89f2bd10 100644 --- a/.claude/shared/run-layout.md +++ b/.claude/shared/run-layout.md @@ -12,6 +12,7 @@ runs/////{task.json, task.log, artifacts/} - `` — zero-padded replicate index (e.g. `00`, `01`). - `task.json` — the persisted per-replicate result (the consumer contract; carries the large `iterations` array — still accepted under its former name `turns` when reading, but not what current runs write). - `task.json.malformed` — present only on the docker degrade path: when an existing `task.json` fails to parse (schema skew from a stale `:latest` image, or a truncated/torn write), the docker runner moves the unparseable original aside to this sidecar and writes a synthetic `final_status=ERROR` `task.json` in its place. Diagnostic-only; `rglob("task.json")` consumers do not match it. +- `task.execute.json` — present only after a DETACHED grade (`coder-eval evaluate ` or `coder-eval run --resume` over a `NOT_GRADED` row). The pre-grade snapshot of `task.json`, written once and never overwritten by a later grade, so "this run was executed separately from grading" stays auditable. Diagnostic-only; `rglob("task.json")` consumers do not match it. - `task.log` — the human-readable task log; `artifacts/` — files the agent produced. **Scope-marker files** (used to detect what a given path represents): diff --git a/CLAUDE.md b/CLAUDE.md index 2b8ac212..71e487f3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -223,7 +223,7 @@ make plugin-reference # the plugin's bundled criteria reference from the models Editing `src/coder_eval/pricing.py` means editing `evalboard/lib/pricing.ts` too — it is a hand-copied mirror, and `evalboard/lib/__tests__/pricing-parity.test.ts` fails the build on drift in either direction. -Recent additions, each traceable to a shipped defect: **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's). +Recent additions, each traceable to a shipped defect: **CE047** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's). When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033–CE036 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE036 enforces the `live_verdict` determinism + monotonicity contract (`criteria/base.py`) that `EarlyStopWatcher`'s latching, deferred fail-stop, and flip-attribution silently depend on: monotonicity over arbitrary Python is undecidable, so instead of a static check it REPLAYS each live criterion against every prefix of recorded trajectories (`tests/lint/live_verdict_contract.py::CASES`) — on the authored ordering AND under seeded shuffles (`permuted_violations`, which catch order-sensitive bugs the authored walk misses) — and asserts the property directly, plus registry-derived coverage — every `LiveSuccessCriterion` in the union must have cases, and every polarity its instances claim via `live_decidable_polarities()` must actually be reached by one (otherwise a single always-`undecided` fixture would "cover" a type while proving nothing). Adding a live criterion therefore means adding `ContractCase`s in the same change. CE035 resolves every `steps..outputs.` / `needs..outputs.` reference in `.github/workflows/**` to a writer that actually produces that key — GitHub expands an unwritten output to the empty string, so a typo degrades a gate silently and actionlint models `steps.*.outputs` as an open string map. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.) diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index b4c5dce0..c9405ae6 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -23,6 +23,7 @@ read). Times are ISO-8601. | --- | --- | --- | | `run.json` / `run.md` | `RunSummary` | Every run (and rebuildable via `coder-eval aggregate`) | | `///task.json` | `EvaluationResult` | One per replicate | +| `///task.execute.json` | `EvaluationResult` | Pre-grade snapshot, written once by a detached grade (`evaluate ` / `run --resume`). Deliberately **not** matched by `rglob("task.json")`, so it never enters an aggregation. | | `//suite.json` / `.md` | `SuiteRollup` | Dataset-backed suites only | | `experiment.json` / `.md` | `ExperimentResult` | Every run (experiment layer) | | `/variant.json` / `.md` | `VariantAggregate` | Per variant | @@ -44,7 +45,7 @@ run-level summary; full per-replicate detail lives in each `task.json`. | `start_time` / `end_time` | `datetime` | Run window. | | `total_duration_seconds` | `float` | Wall-clock. | | `tasks_run` | `int` | Total replicates executed. | -| `tasks_succeeded` / `tasks_failed` / `tasks_error` / `tasks_not_graded` | `int` | Category counts. **Invariant:** the four sum to `tasks_run`. | +| `tasks_succeeded` / `tasks_failed` / `tasks_error` | `int` | Category counts. **Invariant:** these three plus `tasks_not_graded` sum to `tasks_run`. | | `tasks_not_graded` | `int` | Tasks run by `coder-eval execute` — executed, deliberately unscored. Excluded from **both** sides of `pass_rate`. Defaults to `0`, so pre-`execute` `run.json` still parses. | | `tasks_token_budget_exceeded` / `tasks_cost_budget_exceeded` | `int` | Sub-counters of `tasks_failed` (not part of the invariant). | | `skipped_tasks` | `list[{path, reason}]` | Load failures / `skip: true` opt-outs. | @@ -265,8 +266,8 @@ Written for dataset-backed suites; its `passed` flag drives the CI exit code. | Key | Type | Meaning | | --- | --- | --- | | `suite_id` / `variant_id` | `str` | Identity. | -| `rows_total` / `rows_passed` / `rows_failed` / `rows_error` | `int` | Row counts. | -| `pass_rate` | `float` | `rows_passed / rows_total`. | +| `rows_total` / `rows_passed` / `rows_failed` / `rows_error` / `rows_not_graded` | `int` | Row counts. **Invariant:** the four category counts sum to `rows_total`. `rows_not_graded` defaults to `0`. | +| `pass_rate` | `float` | `rows_passed / (rows_total - rows_not_graded)` — ungraded rows leave both sides, matching `RunSummary.pass_rate`. | | `average_weighted_score` | `float \| null` | Mean row score. | | `criterion_stats` | `list[{criterion_type, rows_evaluated, average_score, error_count}]` | Per-criterion summary. | | `failed_samples` | `list[FailedRowSummary]` | Capped at 20 (`{row_id, task_id, final_status, weighted_score, failure_reasons, error_message, task_json_relpath, replicate_index}`). | diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index c37fa46f..bbd65286 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -80,7 +80,7 @@ you want to iterate on afterwards. Grade the results later with budget breach still reports `ERROR` / `TIMEOUT` / `TOKEN_BUDGET_EXCEEDED` and still exits non-zero, exactly as under `run`. -Every `run` flag is available except three things, each refused rather than quietly +Every `run` flag is available except two things, each refused rather than quietly degraded: | Not supported | Why | @@ -127,6 +127,14 @@ original-config results, so the run genuinely mixes configs. The `grade` flag is exempt from that warning, because `execute` → `run --resume` is a supported flow rather than a config mistake. +**A dataset task must pin its sample to be resumable.** Stratified sampling +(`--sample-per-stratum` / `dataset.sample_per_stratum`) re-draws on every +invocation, and each row is its own task (`/`) with its own run +directory. A resume therefore draws a *different* row set, finds no `task.json` +for it, and pays for the agent a second time while the executed rows sit +orphaned in the run dir. Set `dataset.sample_seed`, or use `--sample N` (which is +seeded), before splitting a dataset run across `execute` and `run --resume`. + ### `coder-eval plan` — validate tasks ```bash diff --git a/plugins/coder-eval/reference/run-layout.md b/plugins/coder-eval/reference/run-layout.md index a080dedb..2d4a5e65 100644 --- a/plugins/coder-eval/reference/run-layout.md +++ b/plugins/coder-eval/reference/run-layout.md @@ -11,6 +11,7 @@ runs/////{task.json, task.log, artifacts/} - `` — zero-padded replicate index (e.g. `00`, `01`). - `task.json` — the persisted per-replicate result (the consumer contract; carries the large `iterations` array — still accepted under its former name `turns` when reading, but not what current runs write). - `task.json.malformed` — present only on the docker degrade path: when an existing `task.json` fails to parse (schema skew from a stale `:latest` image, or a truncated/torn write), the docker runner moves the unparseable original aside to this sidecar and writes a synthetic `final_status=ERROR` `task.json` in its place. Diagnostic-only; `rglob("task.json")` consumers do not match it. +- `task.execute.json` — present only after a DETACHED grade (`coder-eval evaluate ` or `coder-eval run --resume` over a `NOT_GRADED` row). The pre-grade snapshot of `task.json`, written once and never overwritten by a later grade, so "this run was executed separately from grading" stays auditable. Diagnostic-only; `rglob("task.json")` consumers do not match it. - `task.log` — the human-readable task log; `artifacts/` — files the agent produced. **Scope-marker files** (used to detect what a given path represents): diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index 8c61afc9..d962b050 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -20,11 +20,10 @@ parse_agent_config, ) from ..orchestration.regrade import ( - PRE_GRADE_JSON, - TASK_JSON, RegradeError, back_up_pre_grade_record, default_workspace, + grading_sandbox_config, load_prior_result, regrade_in_place, task_from_prior, @@ -32,6 +31,7 @@ ) from ..orchestration.task_loader import load_task from ..orchestrator import Orchestrator +from ..path_utils import PRE_GRADE_JSON_FILENAME, TASK_JSON_FILENAME, write_text_atomic from ..sandbox import Sandbox from .console import console from .evaluate_target import EvaluateMode, EvaluateTarget, EvaluateTargetError, resolve_evaluate_target @@ -117,6 +117,11 @@ def _resolve_run_dir_or_work_dir(target: EvaluateTarget, workspace: Path | None) if prior is not None: verify_reference_unchanged(prior, task, task_file) + # Snapshot the ungraded record BEFORE anything grades. Taking it inside + # _write_back instead would capture an ALREADY-GRADED record whenever + # --run-dir points at the target run dir (the orchestrator writes there + # first), destroying the very evidence the copy exists to preserve. + back_up_pre_grade_record(target.target) return _ResolvedInputs( target=target, @@ -143,13 +148,18 @@ def _replicate_index_of(run_dir: Path) -> int: def evaluate_command( task_or_run_dir: Path = typer.Argument( # noqa: B008 ..., - metavar="[TASK_FILE] TARGET", + # One metavar per positional, so the usage line reads as Click renders + # it. A composite metavar on the first ("[TASK_FILE] TARGET") plus an + # empty one on the second produced `[TASK_FILE] TARGET []`. + metavar="TASK_FILE_OR_RUN_DIR", help="Task YAML file, or (when it is the only argument) a finished run directory.", exists=True, ), work_dir: Path | None = typer.Argument( # noqa: B008 None, - metavar="", + # No metavar="" here: an empty one leaks a bare `[]` into both the usage + # line and the arguments table. The first positional's metavar already + # spells out the two shapes. help="Directory containing the code to evaluate. Omit when TASK_FILE is a run directory.", ), workspace: Path | None = typer.Option( # noqa: B008 @@ -179,7 +189,11 @@ def evaluate_command( True, "--preserve/--no-preserve", "-p/-P", - help="Move sandbox artifacts to run directory (default: preserve). The temp sandbox is always removed.", + help=( + "Move sandbox artifacts to run directory (default: preserve). The temp sandbox is " + "always removed. Ignored when grading in place (the default for a run directory) — " + "an adopted directory is never moved or deleted." + ), ), run_dir: Path | None = typer.Option( # noqa: B008 None, @@ -260,15 +274,11 @@ def run_evaluation( console.print(f"[red]✗ Failed to prepare run directory:[/red] {e}") raise typer.Exit(1) from e - sandbox_config = task.sandbox.model_copy(deep=True) + sandbox_config = grading_sandbox_config(task) if not grade_in_place: # Copy path: preload the sandbox with the work dir as a template source. template_source = TemplateDirSource(path=str(graded_dir.resolve())) sandbox_config.template_sources = [template_source, *(sandbox_config.template_sources or [])] - # Grading never runs a container: the docker driver dispatches through - # DockerRunner, which needs an agent. Force tempdir so a task whose YAML says - # `driver: docker` is still gradeable on the host. - sandbox_config = sandbox_config.model_copy(update={"driver": "tempdir"}) task_dir = task_file.parent.resolve() if task_file is not None else None sandbox = Sandbox(sandbox_config, task_id=task.task_id, task_dir=task_dir) @@ -361,7 +371,19 @@ async def _setup_and_run() -> EvaluationResult: f"[dim]Re-graded {prior.final_status.value} → {result.final_status.value} " + f"over {len(result.iterations)} recorded turn(s).[/dim]" ) - _write_back(target.target, result) + if result.final_status is FinalStatus.ERROR: + # A grading-time crash (a failing checker, an unreachable judge) is + # not a verdict about the run. Writing it back would replace a + # perfectly re-gradeable NOT_GRADED row with ERROR — which BOTH + # commands treat as permanently complete, so the run could never be + # graded again without hand-restoring task.execute.json. The + # diagnostic row is still in this grade's own run dir. + console.print( + f"[yellow]⚠[/] Grading errored; leaving {target.target / TASK_JSON_FILENAME} " + + "as it was so the run stays re-gradeable." + ) + else: + _write_back(target.target, result) if result.final_status == FinalStatus.ERROR: console.print(f"\n[red]✗ Evaluation error: {result.error_message}[/red]") @@ -385,11 +407,19 @@ def _write_back(run_dir: Path, result: EvaluationResult) -> None: ungraded record is auditable — the write is not a silent overwrite of the only evidence that the run was executed separately. """ - target = run_dir / TASK_JSON - backup = run_dir / PRE_GRADE_JSON - back_up_pre_grade_record(run_dir) + target = run_dir / TASK_JSON_FILENAME + backup = run_dir / PRE_GRADE_JSON_FILENAME + if target.is_symlink(): + # A run directory is a shareable artifact, so its task.json is untrusted + # input. Following a symlink here turns `evaluate ` into an + # arbitrary-file-overwrite primitive on the grader's host. + console.print(f"[yellow]⚠[/] {target} is a symlink; refusing to write through it.") + return try: - target.write_text(result.model_dump_json(indent=2), encoding="utf-8") + # Atomic, matching the orchestrator's own task.json writer: a torn write + # here makes the row parse as malformed, which a later --resume reads as + # "not complete" and re-pays for the agent. + write_text_atomic(target, result.model_dump_json(indent=2)) except OSError as e: # Never fail the grade over the write-back: the verdict was computed and # already printed, and the fresh run dir holds its own task.json. diff --git a/src/coder_eval/cli/evaluate_target.py b/src/coder_eval/cli/evaluate_target.py index d8039a4e..f9b72b04 100644 --- a/src/coder_eval/cli/evaluate_target.py +++ b/src/coder_eval/cli/evaluate_target.py @@ -21,8 +21,7 @@ from enum import StrEnum from pathlib import Path - -TASK_JSON = "task.json" +from ..path_utils import TASK_JSON_FILENAME class EvaluateMode(StrEnum): @@ -53,7 +52,7 @@ class EvaluateTargetError(ValueError): def is_run_dir(path: Path) -> bool: """Whether ``path`` is a finished task run directory (it holds ``task.json``).""" - return (path / TASK_JSON).is_file() + return (path / TASK_JSON_FILENAME).is_file() def resolve_evaluate_target(first: Path, second: Path | None) -> EvaluateTarget: @@ -79,12 +78,12 @@ def resolve_evaluate_target(first: Path, second: Path | None) -> EvaluateTarget: if not first.is_dir(): raise EvaluateTargetError( f"{first} is not a directory. With a single argument, pass a finished run " - + f"directory (one containing {TASK_JSON}). To grade a directory against a " + + f"directory (one containing {TASK_JSON_FILENAME}). To grade a directory against a " + "task, pass both: coder-eval evaluate " ) if not is_run_dir(first): raise EvaluateTargetError( - f"{first} holds no {TASK_JSON}, so it is not a run directory. Pass the task " + f"{first} holds no {TASK_JSON_FILENAME}, so it is not a run directory. Pass the task " + f"file too: coder-eval evaluate {first}" ) return EvaluateTarget(mode=EvaluateMode.RUN_DIR, target=first, task_file=None) diff --git a/src/coder_eval/cli/plan_command.py b/src/coder_eval/cli/plan_command.py index 25318717..61d2da8b 100644 --- a/src/coder_eval/cli/plan_command.py +++ b/src/coder_eval/cli/plan_command.py @@ -47,6 +47,19 @@ def plan_command( coder-eval plan tasks/*.yaml coder-eval plan tasks/*.yaml -e experiments/model-comparison.yaml """ + run_plan(task_files=task_files, experiment=experiment) + + +def run_plan(*, task_files: list[Path] | None = None, experiment: Path | None = None) -> None: + """The body of ``coder-eval plan``, with real Python defaults. + + Split from the Typer signature for the same reason as ``run_pipeline`` / + ``run_evaluation``: calling a Typer command function in process hands every + unspecified option an ``OptionInfo`` sentinel rather than its default, and + the sentinel is truthy. The `isinstance(experiment, Path)` guard this + function used to need was that bug being papered over rather than fixed. + Callers (tests, library use) call this. Enforced by lint rule CE048. + """ # Default to discovering all tasks under tasks/ when none provided resolved_task_files = task_files if task_files else discover_default_tasks() @@ -64,7 +77,7 @@ def plan_command( from ..orchestration.run_limits import validate_run_limits # Always load experiment (defaults to experiments/default.yaml) - exp_path = experiment if isinstance(experiment, Path) else DEFAULT_EXPERIMENT_PATH + exp_path = experiment or DEFAULT_EXPERIMENT_PATH try: exp_def = load_experiment(exp_path) if exp_path == DEFAULT_EXPERIMENT_PATH: diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index 5c3de001..be6ee073 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -17,7 +17,7 @@ from ..config import Settings, settings from ..logging_config import setup_logging -from ..models import PreservationMode, ResolvedTask, RunSummary, TaskResult +from ..models import EvaluationResult, FinalStatus, PreservationMode, ResolvedTask, RunSummary, TaskResult from ..orchestration.config import BatchRunConfig from ..path_utils import create_latest_symlink, format_task_log_id from ..streaming.callbacks import CompositeStreamCallback @@ -192,10 +192,16 @@ def run_command( "Resume an interrupted run: skip tasks already finalized in --run-dir and " "run only the rest, folding prior results into run.json. A task counts as " "finalized once it has ANY final status — including FAILED/ERROR — so resume " - "does NOT retry failures (delete a task's task.json to force a re-run). " + "does NOT retry failures (delete a task's task.json to force a re-run). The " + "one exception is a NOT_GRADED row left by `coder-eval execute`: `run` was " + "asked for a verdict, so those rows are GRADED in place against the " + "trajectory and workspace already on disk, without re-running the agent. " "Requires --run-dir. A config mismatch (model/backend/flags) is warned, not " "refused — the resumed tasks keep their original-config results, so the run " - "mixes configs; use a fresh --run-dir to keep configs separate." + "mixes configs; use a fresh --run-dir to keep configs separate. A dataset " + "task using nondeterministic stratified sampling needs dataset.sample_seed " + "(or --sample) to be resumable — otherwise the resume draws a different row " + "set and pays for the agent twice." ), ), max_parallel: int = typer.Option( @@ -678,7 +684,15 @@ async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[Resol A task that cannot be graded is reported and folded back in with its ORIGINAL ungraded result, so one bad row neither aborts the resume nor silently - vanishes from run.json — it stays visible as ``tasks_not_graded``. + vanishes from run.json — it stays visible as ``tasks_not_graded``, with the + reason on its ``error_message``. That covers three shapes: a helper raising, + a row too broken to read at all (skipped entirely — there is nothing to fold + back), and a re-grade that returns ``FinalStatus.ERROR``, which + ``Orchestrator.run()`` produces INSTEAD of raising and which would otherwise + make the row permanently un-regradeable. + + Returns the graded rows; the caller's exit gate fails the command whenever + any row is still ungraded, so a resume that graded nothing never exits 0. """ from ..orchestration.regrade import ( RegradeError, @@ -686,14 +700,20 @@ async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[Resol default_workspace, load_prior_result, regrade_in_place, - verify_reference_unchanged, ) graded: list[tuple[ResolvedTask, TaskResult]] = [] + failed_to_load: list[str] = [] for rt in to_grade: - prior = load_prior_result(rt.run_dir) + # Inside the try: an unreadable row must skip like any other grading + # failure. Outside it, one bad task.json propagates out of the loop and + # aborts the whole resume BEFORE run_batch, so none of the `to_run` + # tasks execute either — the opposite of "one bad row never aborts". + prior: EvaluationResult | None = None try: - verify_reference_unchanged(prior, rt.task, rt.task_file) + prior = load_prior_result(rt.run_dir) + # The reference check lives inside regrade_in_place, so a caller + # cannot forget it. workspace = default_workspace(rt.run_dir, prior) # Preserve the ungraded record BEFORE the orchestrator overwrites # task.json in this same directory. @@ -710,12 +730,33 @@ async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[Resol ) except (RegradeError, OSError, RuntimeError, ValueError) as e: console.print(f"[yellow]⚠[/] Could not grade {rt.task.task_id}: {e}") + if prior is None: + # The row could not even be read, so there is nothing to fold + # back. Skipping keeps it out of run.json exactly as it already + # is on disk, and the exit gate still fails the command because + # a task the resume owed a grade produced none. + failed_to_load.append(rt.task.task_id) + continue # Stamp the reason onto the row. Without it the failure survives only # in this console line: the folded-back result keeps the execute # phase's empty error_message, so run.json, the reports and CI show # an ungraded row with no explanation of why grading never happened. result = prior result.error_message = f"Grading failed during --resume: {e}" + else: + if result.final_status is FinalStatus.ERROR: + # An orchestrator-level grading crash is not a verdict about the + # run. Orchestrator.run() converts internal failures into a + # populated ERROR result rather than raising, so without this the + # `except` above never sees them and the ERROR row replaces a + # perfectly re-gradeable NOT_GRADED one — and ERROR is "complete" + # for both commands, so the row could never be graded again. + console.print( + f"[yellow]⚠[/] Grading {rt.task.task_id} errored ({result.error_message}); " + + "keeping the ungraded row so it stays re-gradeable." + ) + prior.error_message = f"Grading errored during --resume: {result.error_message}" + result = prior graded.append( ( rt, diff --git a/src/coder_eval/cli/run_helpers.py b/src/coder_eval/cli/run_helpers.py index 90324ca2..3c5b74bc 100644 --- a/src/coder_eval/cli/run_helpers.py +++ b/src/coder_eval/cli/run_helpers.py @@ -137,8 +137,17 @@ def print_execution_summary(run_dir: Path, summary: RunSummary) -> None: # happened instead, and keep the graded line for whatever WAS graded. if summary.tasks_not_graded: console.print(f"[bold]Results:[/bold] {summary.tasks_not_graded}/{summary.tasks_run} executed, not graded") - console.print("[dim]Grade later: uv run coder-eval evaluate [/dim]") - if summary.tasks_graded: + # Point at the run-dir form, not `evaluate `: the + # two-argument shape grades a bare directory with NO trajectory, so + # command_executed / skill_triggered / trajectory-reading judges score + # differently from what `run` would have produced. The run-dir form + # restores the trajectory AND the resolved config. + console.print(f"[dim]Grade later: uv run coder-eval run --run-dir {run_dir} --resume[/dim]") + console.print("[dim] or: uv run coder-eval evaluate ///00[/dim]") + if summary.tasks_graded or not summary.tasks_not_graded: + # The `or not ...` keeps the pre-existing "0/0 succeeded" line for an + # empty run: without it a run with no tasks at all prints no Results + # line whatsoever, since both counters are falsy. console.print(f"[bold]Results:[/bold] {summary.tasks_succeeded}/{summary.tasks_graded} succeeded") console.print(f"[dim]View report: open {run_dir / 'experiment.md'}[/dim]") console.print(f"[dim]View report: uv run coder-eval report {run_dir}[/dim]") diff --git a/src/coder_eval/cli/run_task_internal_command.py b/src/coder_eval/cli/run_task_internal_command.py index 8bc885b5..e3c4ad85 100644 --- a/src/coder_eval/cli/run_task_internal_command.py +++ b/src/coder_eval/cli/run_task_internal_command.py @@ -156,7 +156,15 @@ def _watch_host_heartbeat() -> None: # `coder-eval run` vs `coder-eval execute`, decided host-side. Defaults to # True (grade) so a host that predates `execute` — which never writes the # key — keeps its exact behavior. - grade: bool = context.get("grade", True) + # Coerced, not annotated: every other value crossing this boundary goes + # through a validating constructor, but `grade` was taken raw — so a + # hand-edited or older-format `"grade": "false"` arrives as a truthy str + # typed as bool and silently grades a run that asked not to be graded. + grade_raw = context.get("grade", True) + if not isinstance(grade_raw, bool): + typer.echo(f"FATAL: context.json 'grade' must be a boolean, got {grade_raw!r}", err=True) + raise typer.Exit(2) + grade: bool = grade_raw # Docker WORKDIR alignment: the host resolves the concrete WORKDIR # (config value / "auto" -> `docker inspect` / fallback) and forwards it here. # Absent -> None -> standard run_dir/artifacts workspace. diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py index c399d7bf..d6b8745a 100644 --- a/src/coder_eval/models/experiment.py +++ b/src/coder_eval/models/experiment.py @@ -233,7 +233,10 @@ class VariantAggregate(BaseModel): # noqa: CE009 -- persisted result model; rou ge=0, description="Tasks executed without grading (`coder-eval execute`). Excluded from pass_rate entirely.", ) - average_score: float + # None when nothing in this variant was graded (`coder-eval execute`). + # A 0.0 here is indistinguishable from "measured and scored zero" — the same + # reason EvaluationResult.weighted_score is Optional. + average_score: float | None average_duration: float total_tokens: int | None = None replicate_count: int = Field( diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index 24d0effe..58cc3558 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -829,10 +829,10 @@ def _pick_worst_status(statuses: list[FinalStatus]) -> FinalStatus: return min(statuses, key=lambda s: priority.get(s.category, -1)) -def _mean_graded_score(vr_list: list[VariantResult]) -> float: - """Mean ``weighted_score`` over the graded rows; ``0.0`` when none were graded.""" +def _mean_graded_score(vr_list: list[VariantResult]) -> float | None: + """Mean ``weighted_score`` over the graded rows; ``None`` when none were graded.""" graded = [v.weighted_score for v in vr_list if v.weighted_score is not None] - return sum(graded) / len(graded) if graded else 0.0 + return sum(graded) / len(graded) if graded else None def _mean_reference_similarity(reps: list[TaskResult]) -> float | None: @@ -943,7 +943,7 @@ def aggregate_results( tasks_succeeded=0, tasks_failed=0, tasks_error=0, - average_score=0.0, + average_score=None, average_duration=0.0, ) continue @@ -959,10 +959,9 @@ def aggregate_results( tasks_not_graded=sum(1 for v in vr_list if v.final_status.category == "ungraded"), tasks_token_budget_exceeded=sum(1 for v in vr_list if v.final_status == FinalStatus.TOKEN_BUDGET_EXCEEDED), tasks_cost_budget_exceeded=sum(1 for v in vr_list if v.final_status == FinalStatus.COST_BUDGET_EXCEEDED), - # Mean over GRADED rows only. An ungraded row has no score (it - # arrives here as 0.0 because VariantResult.weighted_score is a - # plain float), so including it would report a clean execute run as - # average_score 0.0 — a number indistinguishable from "scored zero". + # Mean over GRADED rows only, and None when there are none: a clean + # execute run has no average score, and reporting 0.000 next to + # "Pass Rate: n/a" is a number indistinguishable from "scored zero". average_score=_mean_graded_score(vr_list), average_duration=sum(v.duration_seconds / v.replicate_count for v in vr_list) / len(vr_list), total_tokens=total_tokens, diff --git a/src/coder_eval/orchestration/regrade.py b/src/coder_eval/orchestration/regrade.py index 8eba319d..892cee0e 100644 --- a/src/coder_eval/orchestration/regrade.py +++ b/src/coder_eval/orchestration/regrade.py @@ -20,14 +20,19 @@ import logging from pathlib import Path -from coder_eval.models import EvaluationResult, PreservationMode, TaskConfigRecord, TaskDefinition +from coder_eval.models import ( + EvaluationResult, + PreservationMode, + SandboxConfig, + TaskConfigRecord, + TaskDefinition, +) +from coder_eval.path_utils import PRE_GRADE_JSON_FILENAME, TASK_JSON_FILENAME from coder_eval.sandbox import Sandbox logger = logging.getLogger(__name__) -TASK_JSON = "task.json" -PRE_GRADE_JSON = "task.execute.json" ARTIFACTS_DIRNAME = "artifacts" @@ -37,7 +42,7 @@ class RegradeError(Exception): def load_prior_result(run_dir: Path) -> EvaluationResult: """Read a finished run's ``task.json``.""" - path = run_dir / TASK_JSON + path = run_dir / TASK_JSON_FILENAME try: raw = path.read_text(encoding="utf-8") except OSError as e: @@ -64,7 +69,7 @@ def task_from_prior(prior: EvaluationResult, run_dir: Path) -> tuple[TaskDefinit record = prior.task_config if record is None: raise RegradeError( - f"{run_dir / TASK_JSON} carries no task_config, so the executed task cannot be " + f"{run_dir / TASK_JSON_FILENAME} carries no task_config, so the executed task cannot be " + "rebuilt. Pass the task file explicitly: coder-eval evaluate " ) try: @@ -104,7 +109,7 @@ def _fall_back_to_source(record: TaskConfigRecord, run_dir: Path, e: ValueError) if not record.source_file or not Path(record.source_file).is_file(): raise RegradeError( - f"The resolved task config in {run_dir / TASK_JSON} no longer validates ({e}), and " + f"The resolved task config in {run_dir / TASK_JSON_FILENAME} no longer validates ({e}), and " + "its source YAML is unavailable. Pass the task file explicitly." ) from e logger.warning( @@ -123,12 +128,26 @@ def default_workspace(run_dir: Path, prior: EvaluationResult) -> Path: """Locate the workspace a finished run left behind. ``sandbox_path`` is authoritative when it still exists — it is where the run - actually worked. Otherwise fall back to the preserved artifacts tree, whose - single child is named for the task. + actually worked. Otherwise fall back to the preserved artifacts tree, where + preservation nests the workspace under the task id. + + Raises rather than guessing when neither is conclusive. Guessing is worse + than failing here: grading the WRONG directory makes every path-relative + criterion fail as a locating artifact rather than as a verdict, and it + reports that as an ordinary score. """ if prior.sandbox_path: recorded = Path(prior.sandbox_path) if recorded.is_dir(): + if not _is_within(recorded, run_dir): + # An absolute path out of the run's own task.json, which is + # untrusted input for a shared run dir. Criteria execute with + # cwd there and may mutate it, so an out-of-tree location has to + # be the operator's explicit choice. + raise RegradeError( + f"The recorded sandbox_path ({recorded}) is outside the run directory " + + f"({run_dir}). Pass --workspace explicitly to grade it." + ) return recorded artifacts = run_dir / ARTIFACTS_DIRNAME @@ -138,10 +157,33 @@ def default_workspace(run_dir: Path, prior: EvaluationResult) -> Path: + f"({prior.sandbox_path or 'unset'}) is gone. The run was probably made with " + "--preservation-mode NONE." ) + # The exact path, not a heuristic. `task_id` may contain "/" (dataset rows + # are "/"), so "the single child of artifacts/" resolves one + # level too high for every row task. + by_task_id = artifacts / prior.task_id + if by_task_id.is_dir(): + return by_task_id + children = [p for p in sorted(artifacts.iterdir()) if p.is_dir()] - # Preservation nests the workspace under the task id; a flat artifacts dir - # (no subdirectory) means the workspace IS artifacts/. - return children[0] if len(children) == 1 else artifacts + if not children: + # A flat artifacts dir (no subdirectory) means the workspace IS artifacts/. + return artifacts + if len(children) == 1: + return children[0] + raise RegradeError( + f"Cannot tell which directory under {artifacts} is the workspace: no {prior.task_id!r} " + + f"child, and {len(children)} candidates ({', '.join(p.name for p in children)}). " + + "Pass --workspace explicitly." + ) + + +def _is_within(candidate: Path, root: Path) -> bool: + """True when ``candidate`` resolves inside ``root``.""" + try: + candidate.resolve().relative_to(root.resolve()) + except ValueError: + return False + return True def verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition, task_file: Path | None) -> None: @@ -198,9 +240,14 @@ def back_up_pre_grade_record(run_dir: Path) -> None: a second grade must not overwrite the ORIGINAL execute record with an already-graded one. """ - source, backup = run_dir / TASK_JSON, run_dir / PRE_GRADE_JSON + source, backup = run_dir / TASK_JSON_FILENAME, run_dir / PRE_GRADE_JSON_FILENAME if backup.exists() or not source.is_file(): return + if source.is_symlink() or backup.is_symlink(): + # Untrusted run dir: writing through a symlink would let a shared + # artifact clobber an arbitrary file the grading user can write. + logger.warning("Not preserving the pre-grade record: %s or %s is a symlink.", source, backup) + return try: backup.write_text(source.read_text(encoding="utf-8"), encoding="utf-8") except OSError as e: @@ -208,6 +255,21 @@ def back_up_pre_grade_record(run_dir: Path) -> None: logger.warning("Could not preserve the pre-grade record at %s: %s", backup, e) +def grading_sandbox_config(task: TaskDefinition) -> SandboxConfig: + """The sandbox config a grading pass runs under. + + Grading never runs a container: the docker driver dispatches through + DockerRunner, which needs an agent. Forcing ``tempdir`` keeps a task whose + YAML says ``driver: docker`` gradeable on the host. + + Re-validated rather than ``model_copy(update=...)``: ``update`` skips both + pydantic validation and pyright, so a typo would produce a SandboxConfig + violating its own ``Literal`` and surface much later at an unrelated + ``if driver == "docker"`` branch. + """ + return SandboxConfig.model_validate({**task.sandbox.model_dump(), "driver": "tempdir"}) + + async def regrade_in_place( *, task: TaskDefinition, @@ -232,12 +294,13 @@ async def regrade_in_place( """ from coder_eval.orchestrator import Orchestrator - # Grading never runs a container: the docker driver dispatches through - # DockerRunner, which needs an agent. Force tempdir so a task whose YAML says - # `driver: docker` is still gradeable on the host. - sandbox_config = task.sandbox.model_copy(deep=True).model_copy(update={"driver": "tempdir"}) + # Inside the shared entry point, not at each caller: a guard a caller has to + # remember is one a third caller will forget, and this one is the difference + # between a verdict and a verdict against the wrong answer key. + verify_reference_unchanged(prior, task, task_file) + sandbox = Sandbox( - sandbox_config, + grading_sandbox_config(task), task_id=task.task_id, task_dir=task_file.parent.resolve() if task_file is not None else None, ) diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 1e5c2540..306f3155 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -67,7 +67,13 @@ from .orchestration.early_stop import EarlyStopWatcher, early_stop_active, validate_early_stop from .orchestration.evaluation import resolve_reference_dir, stage_reference_dir from .orchestration.run_limits import validate_run_limits -from .path_utils import digest_tree, format_task_log_id, rmtree_restrictive, task_log_path +from .path_utils import ( + digest_tree, + format_task_log_id, + rmtree_restrictive, + task_log_path, + write_text_atomic, +) from .sandbox import Sandbox from .simulation import DialogStopReason, SimulatorResult, UserSimulator, evaluate_stop from .streaming.callbacks import CompositeStreamCallback, StreamCallback, TaskScopedCallback, safe_emit @@ -763,6 +769,11 @@ def _seed_from_prior_result(self) -> None: # quietly wrong. _finalize_result restores the duration after its own # timing write; the grading pass's cost is recorded separately there. self.result.started_at = prior.started_at + # completed_at too, so the row's three time fields stay consistent with + # each other: leaving it at grading wall-clock produces a triple where + # completed_at - started_at != duration_seconds, which misleads anyone + # deriving a duration from the timestamps. + self.result.completed_at = prior.completed_at # The trajectory itself. Every derived figure in _finalize_result — # token totals, cost, command_stats, model_used, assistant turns — @@ -802,11 +813,25 @@ def _seed_from_prior_result(self) -> None: # environment_info: the prior run's capture describes the machine that # RAN the task (installed_tools, api route, coder_eval version). Ours # describes the machine grading it. Prior wins on conflict, and ours is - # preserved wholesale under `graded_by` rather than being interleaved — + # preserved as flat `graded_by_*` scalars rather than being interleaved — # a report that shows the grader's tool versions as the run's is worse # than one that shows neither. - graded_by = dict(self.result.environment_info) - self.result.environment_info = {**graded_by, **prior.environment_info, "graded_by": graded_by} + # Flattened to scalars rather than nested wholesale: environment_info is + # a flat map everywhere it is consumed (the HTML report `_esc`apes each + # value into a table cell; the evalboard types it as + # Record>), so a + # whole nested env capture renders as a Python dict repr. Only the three + # facts that identify the grading HOST are kept, and only when they + # differ from the run's. + grader = self.result.environment_info + provenance = { + f"graded_by_{key}": grader[key] + for key in ("coder_eval", "git_commit", "cli_version") + if key in grader and grader.get(key) != prior.environment_info.get(key) + } + # A second grade must not lose the first grader's stamp — merging prior + # over ours would otherwise clobber it and collapse the chain silently. + self.result.environment_info = {**grader, **prior.environment_info, **provenance} async def _run_evaluation_with_failure_evidence( self, @@ -1111,10 +1136,8 @@ def _finalize_result(self, start_time: float) -> None: # docker-driver host-heartbeat watchdog firing) would otherwise leave # a truncated task.json that the host parses as malformed-JSON rather # than as "no result", conflating two distinct failure modes. - import os as _os - - report_tmp = self.report_path.with_suffix(self.report_path.suffix + ".tmp") - report_tmp.write_text( # noqa: CE002 — small JSON write at end of run + write_text_atomic( # noqa: CE002 — small JSON write at end of run + self.report_path, self.result.model_dump_json( indent=2, # Strip inline transcripts: they live in sibling YAML files @@ -1123,9 +1146,7 @@ def _finalize_result(self, start_time: float) -> None: # in the row record without losing any data. exclude=TASK_JSON_TRANSCRIPT_EXCLUDE, ), - encoding="utf-8", ) - _os.replace(report_tmp, self.report_path) # Also emit an HTML trace/report alongside task.json. HTML failure must # never mask the underlying run outcome — write_task_html logs and diff --git a/src/coder_eval/path_utils.py b/src/coder_eval/path_utils.py index 44e47d91..c0a3b39f 100644 --- a/src/coder_eval/path_utils.py +++ b/src/coder_eval/path_utils.py @@ -15,6 +15,17 @@ TASK_LOG_FILENAME = "task.log" +# The per-task result record, and the pre-grade snapshot a detached grade keeps +# beside it. Module-level because ~12 sites name them — including three that +# `rglob` for the first — and two half-copies of the same string in different +# packages is how a rename becomes a silent no-op on the sites it missed. +TASK_JSON_FILENAME = "task.json" +PRE_GRADE_JSON_FILENAME = "task.execute.json" + +# The virtualenv directory `setup` creates and `adopt` discovers. Named because +# whether it is on PATH decides which binaries a criterion resolves. +VENV_DIRNAME = ".venv" + # Ignore list for every copy of a reference solution tree. A module-level # constant, not an inline literal at each call site: the host-side docker mount # (`DockerRunner._prepare_reference_mount`) and the per-run staged copy @@ -25,6 +36,22 @@ REFERENCE_COPY_IGNORE = [".git"] +def write_text_atomic(path: Path, text: str) -> None: + """Write ``text`` to ``path`` via a temp file + ``os.replace``. + + A plain ``write_text`` truncates first, so a SIGKILL or a full disk mid-write + leaves a half-file. For ``task.json`` that is worse than no file: a truncated + record parses as *malformed*, which the recovery paths treat as "not + complete" — so a later ``--resume`` re-executes the task and pays for the + agent again, and the row vanishes from ``run.json``. One writer, so the + orchestrator and the detached grade's write-back cannot have different crash + semantics for the same file. + """ + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(text, encoding="utf-8") + os.replace(tmp, path) + + def digest_tree(root: Path) -> str: """Content hash of every file under ``root``, stable across runs. diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index ea1aceac..a10fb5f7 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -27,6 +27,7 @@ fmt_mean_sd, fmt_p, format_score, + is_env_table_key, load_variant_eval_results, paired_comparison, stddev, @@ -632,7 +633,7 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir: # Denominator is the GRADED count, matching VariantAggregate.pass_rate — # an ungraded task was never measured and belongs on neither side. f"- **Pass Rate**: {pass_rate_str} ({agg.tasks_succeeded}/{agg.tasks_graded})", - f"- **Average Score**: {agg.average_score:.3f}", + f"- **Average Score**: {format_score(agg.average_score)}", f"- **Average Duration**: {agg.average_duration:.1f}s", f"- **Total Tokens**: {tokens_str}", ] @@ -723,7 +724,7 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir: # Environment (from first result with data) for er in eval_results: if er.environment_info: - env = {k: v for k, v in er.environment_info.items() if k != "installed_tools"} + env = {k: v for k, v in er.environment_info.items() if is_env_table_key(k)} if env: lines.extend(["", "## Environment", ""]) for key, value in env.items(): diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py index 128c24ed..39e7a600 100644 --- a/src/coder_eval/reports_html.py +++ b/src/coder_eval/reports_html.py @@ -20,7 +20,7 @@ from coder_eval.models import FinalStatus, eval_result_total_cost, sum_costs from .reports import early_stop_gate_note -from .reports_stats import format_score +from .reports_stats import format_score, is_env_table_key if TYPE_CHECKING: @@ -1074,8 +1074,8 @@ def _render_simulation(result: EvaluationResult) -> str: def _render_environment(result: EvaluationResult) -> str: - """Render Environment section (excluding installed_tools, which has its own).""" - env = {k: v for k, v in (result.environment_info or {}).items() if k != "installed_tools"} + """Render Environment section (excluding the keys with their own treatment).""" + env = {k: v for k, v in (result.environment_info or {}).items() if is_env_table_key(k)} if not env: return "" rows = "".join(f"{_esc(k)}{_esc(v)}" for k, v in env.items()) diff --git a/src/coder_eval/reports_stats.py b/src/coder_eval/reports_stats.py index c123764b..78306b5c 100644 --- a/src/coder_eval/reports_stats.py +++ b/src/coder_eval/reports_stats.py @@ -310,6 +310,19 @@ class VariantSeries(NamedTuple): asst_turns: list[float] +# environment_info keys the Environment table must NOT render as ordinary rows. +# `installed_tools` has its own dedicated section; the rest are harness +# bookkeeping the reader did not ask for — `command_base_path` is a full PATH +# string on every row, and the graded_by_* provenance keys only appear on a +# re-graded row where they would read as facts about the run itself. +ENV_TABLE_EXCLUDE = frozenset({"installed_tools", "command_base_path", "reference_digest"}) + + +def is_env_table_key(key: str) -> bool: + """Whether ``key`` belongs in a rendered Environment table.""" + return key not in ENV_TABLE_EXCLUDE and not key.startswith("graded_by_") + + # What an ungraded row shows where a score would go. Deliberately not "0.000": # an ungraded task was never measured, and a zero is indistinguishable from a # task that was measured and scored nothing. diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 51734c69..b77bdb60 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -23,6 +23,7 @@ StarterFilesSource, TemplateDirSource, ) +from .path_utils import VENV_DIRNAME from .resources import get_ignore_patterns, should_ignore_path @@ -298,6 +299,11 @@ def adopt(self, workspace: Path) -> Path: copy path they see the copy's paths, not the ones the agent worked at. * Copying a real workspace costs minutes. + "Materializing nothing" means it writes no FILES. It does still chmod + ``+x`` over the task's declared mock-PATH directories inside the tree — + a mode change the criteria need in order to resolve the same shimmed + binaries the agent did. + The caller keeps ownership: ``_cleanup_on_exit`` stays False, so ``cleanup()`` never deletes an adopted directory. Criteria CAN still mutate it (a ``run_command`` that writes), which is why the copy path @@ -341,9 +347,17 @@ def adopt(self, workspace: Path) -> Path: # Discover an existing venv instead of creating one, so `run_command` # criteria get the same VIRTUAL_ENV/PATH the agent had. Absent venv -> # None, exactly as for a task with no python config. - candidate = self.sandbox_dir / ".venv" - if candidate.is_dir(): - self.venv_dir = candidate + # + # Gated on `config.python` for the same reason `setup` is: venv_dir + # prepends the venv's bin/ to PATH and exports VIRTUAL_ENV for every + # criterion subprocess, so discovering one a task never asked for grades + # it under a PATH it never ran under — the exact divergence the + # command_base_path round trip exists to close. It would also let an + # agent shadow binaries by writing `.venv/bin/` into its own workspace. + if self.config.python: + candidate = self.sandbox_dir / VENV_DIRNAME + if candidate.is_dir(): + self.venv_dir = candidate self._check_parent_node_modules_contamination() self._refresh_plugin_tools_dir() @@ -800,7 +814,7 @@ def _setup_virtualenv(self) -> None: if not self.sandbox_dir: raise RuntimeError("Sandbox directory not initialized") - self.venv_dir = self.sandbox_dir / ".venv" + self.venv_dir = self.sandbox_dir / VENV_DIRNAME # Use uv to create virtual environment (faster than venv) try: diff --git a/tests/lint/rules/ce047_env_info_key_round_trip.py b/tests/lint/rules/ce047_env_info_key_round_trip.py new file mode 100644 index 00000000..1afe6281 --- /dev/null +++ b/tests/lint/rules/ce047_env_info_key_round_trip.py @@ -0,0 +1,122 @@ +"""CE047: every ``environment_info`` key that is READ must also be WRITTEN. + +``EvaluationResult.environment_info`` is a ``dict[str, Any]`` bag, so nothing — +not pydantic, not pyright — connects the site that writes a key to the site that +reads it back. A reader whose writer was never added (or was later removed) is +silently inert: ``.get("k")`` returns ``None``, the guard takes its early return, +and the feature reports success while doing nothing. + +The motivating case: ``verify_reference_unchanged`` read +``environment_info.get("reference_digest")`` to refuse a re-grade whose answer key +had changed. Nothing anywhere wrote that key — a whole-tree grep found exactly one +occurrence, the read itself. The anti-cheat guard shipped, was documented in +CLAUDE.md and the user guide as protection, and never fired once. Every automated +gate in the repo was green. + +This is deliberately a one-way check. An unread key is ordinary (recorded for a +human or a downstream consumer); an unwritten key is always a bug. + +Use ``# noqa: CE047`` for a key genuinely supplied from outside this repo. +""" + +import ast +import re +from pathlib import Path + +from tests.lint.rules.base import BaseRule + + +_SRC_ROOT = Path("src/coder_eval") + +# Keys written by a consumer outside src/ (the docker container's own capture, +# a plugin) or copied wholesale from another dict. Each needs a reason. +_EXTERNALLY_WRITTEN: dict[str, str] = {} + + +def _written_keys() -> set[str]: + """Every string literal assigned into an ``environment_info`` subscript. + + Text-scanned rather than AST-walked across the tree so a write inside any + module counts regardless of how the dict was reached (``self.result.``, + ``result.``, a local alias). The rule only needs to know a literal is + written SOMEWHERE — attributing it precisely would add false positives + without catching anything more. + """ + written: set[str] = set() + pattern = re.compile(r"""environment_info\[\s*["']([\w.-]+)["']\s*\]\s*=""") + # Also count keys named in a dict literal that becomes environment_info, and + # the f-string-built provenance keys (`f"graded_by_{key}"`), which no literal + # scan can resolve — those are covered by the prefix allowance below. + for path in sorted(_SRC_ROOT.rglob("*.py")): + try: + text = path.read_text(encoding="utf-8") + except OSError: # pragma: no cover - unreadable file in src is not our problem + continue + written.update(pattern.findall(text)) + return written + + +class EnvInfoKeyRoundTrip(BaseRule): + id = "CE047" + + _SRC_PATH = re.compile(r"[/\\]src[/\\]coder_eval[/\\]") + _written: set[str] | None = None + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = bool(self._SRC_PATH.search(filepath)) + if self._in_scope and EnvInfoKeyRoundTrip._written is None: + EnvInfoKeyRoundTrip._written = _written_keys() + + def visit_Call(self, node: ast.Call) -> None: + self._check_get(node) + self.generic_visit(node) + + def visit_Subscript(self, node: ast.Subscript) -> None: + self._check_subscript(node) + self.generic_visit(node) + + def _check_get(self, node: ast.Call) -> None: + """``<...>.environment_info.get("key")``.""" + func = node.func + if not isinstance(func, ast.Attribute) or func.attr != "get": + return + if not _is_env_info(func.value) or not node.args: + return + self._require_writer(node, node.args[0]) + + def _check_subscript(self, node: ast.Subscript) -> None: + """``<...>.environment_info["key"]`` in a READ position. + + A write is an ``ast.Store`` context, which is exactly what makes it a + writer — only loads are checked. + """ + if not isinstance(node.ctx, ast.Load) or not _is_env_info(node.value): + return + self._require_writer(node, node.slice) + + def _require_writer(self, node: ast.AST, key_node: ast.AST) -> None: + if not self._in_scope: + return + if not isinstance(key_node, ast.Constant) or not isinstance(key_node.value, str): + return # a computed key; nothing to resolve statically + key = key_node.value + if key in _EXTERNALLY_WRITTEN or key.startswith("graded_by_"): + # graded_by_* keys are built with an f-string from a name list, so no + # literal write exists to find. + return + if key in (EnvInfoKeyRoundTrip._written or set()): + return + self.violation( + node, + f"environment_info key {key!r} is read here but never written anywhere in src/coder_eval. " + + "A reader with no writer is silently inert — it returns None, the guard takes its early " + + "return, and the feature reports success while doing nothing (see CE047's docstring for " + + "the anti-cheat guard that shipped this way). Add the write, or list the key in " + + "_EXTERNALLY_WRITTEN with the out-of-tree producer that supplies it.", + ) + + +def _is_env_info(node: ast.AST) -> bool: + """Whether ``node`` is an ``…​.environment_info`` attribute access.""" + return isinstance(node, ast.Attribute) and node.attr == "environment_info" diff --git a/tests/lint/rules/ce048_no_in_process_typer_command_call.py b/tests/lint/rules/ce048_no_in_process_typer_command_call.py new file mode 100644 index 00000000..37c10a2f --- /dev/null +++ b/tests/lint/rules/ce048_no_in_process_typer_command_call.py @@ -0,0 +1,106 @@ +"""CE048: never call a Typer command function in process. + +Typer builds a command's parser from its signature, so every parameter's default +is an ``OptionInfo`` / ``ArgumentInfo`` sentinel, not the value it stands for. +Click substitutes the real defaults when it *invokes* the command; a direct +Python call does not — every unspecified argument arrives as a truthy sentinel +object. + +The failure is silent, which is what makes it worth a rule. ``evaluate``'s +``in_place: bool | None = typer.Option(None, "--in-place/--copy")`` reads as "no +preference" and selects copy-vs-in-place from the target shape; called +in-process, ``in_place`` was an ``OptionInfo``, which is truthy, so the tests +silently graded in place and the default they meant to cover was never +exercised. Nothing failed — the wrong branch simply ran. + +The fix is the one already applied to ``run`` / ``execute`` / ``evaluate``: keep +the Typer signature as a thin wrapper and put the body in a plain function with +real Python defaults (``run_pipeline``, ``run_evaluation``). Call THAT. + +Use ``# noqa: CE048`` only where the sentinel behavior is itself under test. +""" + +import ast +import re +from pathlib import Path + +from tests.lint.rules.base import BaseRule + + +_CLI_ROOT = Path("src/coder_eval/cli") + + +def _typer_command_names() -> set[str]: + """Functions whose signature is a Typer parser — i.e. whose parameters carry + ``typer.Option`` / ``typer.Argument`` defaults. + + Detected by the defaults rather than by the ``app.command(...)`` registration + site, because registration happens in ``cli/__init__.py`` by reference and a + command that is merely *about* to be registered has the same hazard. + """ + names: set[str] = set() + for path in sorted(_CLI_ROOT.rglob("*.py")): + try: + tree = ast.parse(path.read_text(encoding="utf-8")) + except (OSError, SyntaxError): # pragma: no cover - unparseable file in cli/ fails elsewhere + continue + for node in ast.walk(tree): + if not isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + continue + if any(_is_typer_param(default) for default in node.args.defaults): + names.add(node.name) + return names + + +def _is_typer_param(node: ast.expr) -> bool: + """``typer.Option(...)`` / ``typer.Argument(...)`` as a parameter default.""" + if not isinstance(node, ast.Call): + return False + func = node.func + return isinstance(func, ast.Attribute) and func.attr in {"Option", "Argument"} + + +class NoInProcessTyperCommandCall(BaseRule): + id = "CE048" + + # The registration site itself hands these to Typer by reference; and the + # module that defines a command may call its own sibling. + _EXEMPT_FILES = re.compile(r"[/\\]src[/\\]coder_eval[/\\]cli[/\\]__init__\.py$") + _commands: set[str] | None = None + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = not self._EXEMPT_FILES.search(filepath) + self._imported_from_cli: set[str] = set() + if NoInProcessTyperCommandCall._commands is None: + NoInProcessTyperCommandCall._commands = _typer_command_names() + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + # Only names imported FROM a cli module count. A command name is not + # unique in the tree — `run_command` is both a Typer command and + # `Sandbox.run_command` — so matching on the bare name alone would flag + # every criterion that shells out. + if (node.module and "coder_eval.cli" in node.module) or (node.level and node.module == "cli"): + self._imported_from_cli.update(alias.asname or alias.name for alias in node.names) + self.generic_visit(node) + + def visit_Call(self, node: ast.Call) -> None: + self._check(node) + self.generic_visit(node) + + def _check(self, node: ast.Call) -> None: + if not self._in_scope or not isinstance(node.func, ast.Name): + return + name = node.func.id + if name not in self._imported_from_cli: + return + if name not in (NoInProcessTyperCommandCall._commands or set()): + return + self.violation( + node, + f"'{name}' is a Typer command: its parameter defaults are OptionInfo sentinels, not values, " + + "so calling it in process hands every unspecified argument a truthy placeholder and " + + "silently runs the wrong branch. Call the plain-function body instead (the " + + "run_pipeline / run_evaluation split exists for this), or drive it through " + + "typer.testing.CliRunner.", + ) diff --git a/tests/lint/runner.py b/tests/lint/runner.py index 092e97a6..383c3064 100644 --- a/tests/lint/runner.py +++ b/tests/lint/runner.py @@ -26,6 +26,8 @@ from tests.lint.rules.ce039_config_error_escalates import ConfigErrorEscalates from tests.lint.rules.ce043_no_command_output_truncation import NoCommandOutputTruncation from tests.lint.rules.ce046_env_info_spreads_super import EnvInfoSpreadsSuper +from tests.lint.rules.ce047_env_info_key_round_trip import EnvInfoKeyRoundTrip +from tests.lint.rules.ce048_no_in_process_typer_command_call import NoInProcessTyperCommandCall from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore @@ -75,6 +77,8 @@ ConfigErrorEscalates, NoCommandOutputTruncation, EnvInfoSpreadsSuper, + EnvInfoKeyRoundTrip, + NoInProcessTyperCommandCall, ] # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 64508fe7..8329d629 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -23,13 +23,21 @@ SRC = Path(__file__).parent.parent / "src" +# Rules whose defect class lives in the TEST tree, not in src/. CE048's whole +# subject is an in-process call to a Typer command, and the only place that +# happens is a test — scanning src/ alone would leave the rule permanently green +# while the bug it exists for sat five lines away. +_ALSO_SCAN_TESTS = {"CE048"} + + @pytest.mark.lint @pytest.mark.parametrize("rule_class", ALL_RULES, ids=[r.id for r in ALL_RULES]) def test_no_violations(rule_class: type) -> None: import sys mod_doc = (getattr(sys.modules.get(rule_class.__module__), "__doc__", "") or "").splitlines()[0].strip() - violations = check_paths([SRC], rules=[rule_class]) + paths = [SRC, Path(__file__).parent] if rule_class.id in _ALSO_SCAN_TESTS else [SRC] + violations = check_paths(paths, rules=[rule_class]) assert not violations, ( f"\n{len(violations)} violation(s) for {rule_class.id} ({mod_doc}):\n\n" + "\n".join(f" {v}" for v in violations) diff --git a/tests/test_detached_grading_guards.py b/tests/test_detached_grading_guards.py new file mode 100644 index 00000000..2ff234cf --- /dev/null +++ b/tests/test_detached_grading_guards.py @@ -0,0 +1,261 @@ +"""The guards around detached grading, each tested on the branch that fires. + +Every case here is a refusal, a skip, or a mode selection — the branches that +exist precisely because taking the other one would produce a plausible number +that is wrong. They were all shipped with coverage on the happy path only. +""" + +from __future__ import annotations + +from datetime import datetime +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from coder_eval.cli import app +from coder_eval.cli.evaluate_command import run_evaluation +from coder_eval.models import ( + AgentKind, + EvaluationResult, + FileExistsCriterion, + FinalStatus, + TaskDefinition, + parse_agent_config, +) +from coder_eval.orchestrator import Orchestrator + + +runner = CliRunner() + + +# An agentless task: `type: none` runs no agent and forbids `initial_prompt` +# (there is nothing to read it), which is what makes it usable with no API key. +_AGENTLESS = """task_id: t +description: d +agent: + type: none +success_criteria: + - type: file_exists + path: proof.txt + description: x +""" + +# A simulation task needs a real agent type — the refusal under `execute` fires +# during resolution, so the agent is never created. +_SIMULATED = """task_id: t +description: d +initial_prompt: p +agent: + type: claude-code +simulation: + enabled: true + persona: a user + goal: get it done +success_criteria: + - type: file_exists + path: proof.txt + description: x +""" + + +def _task(tmp_path: Path, *, simulation: bool = False) -> Path: + path = tmp_path / "t.yaml" + path.write_text(_SIMULATED if simulation else _AGENTLESS, encoding="utf-8") + return path + + +# -------------------------------------------------------------------------- +# `execute` refuses simulation tasks +# -------------------------------------------------------------------------- + + +def test_execute_refuses_a_simulation_task_by_name(tmp_path: Path) -> None: + """The dialog loop reads criteria results to decide whether to keep talking, + so an ungraded dialog would silently change its own stopping behavior. The + refusal must name the task, or a user cannot tell which one to remove.""" + result = runner.invoke(app, ["execute", str(_task(tmp_path, simulation=True)), "--run-dir", str(tmp_path / "r")]) + + assert result.exit_code != 0 + assert "simulation" in result.output.lower() + assert "t" in result.output + + +def test_run_still_accepts_the_same_simulation_task(tmp_path: Path) -> None: + """The control: the refusal is about `execute`, not about the task.""" + task = _task(tmp_path, simulation=True) + with patch("coder_eval.cli.run_command._run_with_experiment", new=AsyncMock(return_value=(MagicMock(), 0))): + result = runner.invoke(app, ["run", str(task), "--run-dir", str(tmp_path / "r")]) + assert "does not support simulation" not in result.output + + +# -------------------------------------------------------------------------- +# The evaluate-only path refuses grade=False +# -------------------------------------------------------------------------- + + +async def test_grading_off_on_the_evaluate_only_path_is_refused(tmp_path: Path) -> None: + """No agent AND no grading is a no-op that would still write a task.json. + Refusing beats producing an empty row that looks like a result.""" + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v", grade=False) + orch.success_checker = MagicMock() + orch.result = EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2026, 1, 1), + final_status=FinalStatus.FAILURE, + iteration_count=0, + ) + + with pytest.raises(ValueError, match="meaningless on the evaluate-only path"): + await orch._evaluation_loop() + + +# -------------------------------------------------------------------------- +# --in-place / --copy +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("flag", "expect_adopt"), + [(None, False), ("--in-place", True), ("--copy", False)], + ids=["default-for-a-work-dir-is-copy", "explicit-in-place", "explicit-copy"], +) +def test_the_flag_decides_adopt_versus_setup_on_a_work_dir( + tmp_path: Path, flag: str | None, expect_adopt: bool +) -> None: + """The choice is not cosmetic: the copy path filters node_modules / dist / + build / .venv, so a criterion reading those fails as a copying artifact.""" + work = tmp_path / "work" + work.mkdir() + (work / "proof.txt").write_text("x", encoding="utf-8") + # --run-dir is not incidental: without it the grade lands in a repo-relative + # runs//, which several xdist workers race over. + args = ["evaluate", str(_task(tmp_path)), str(work), "--run-dir", str(tmp_path / "r")] + if flag: + args.append(flag) + + with ( + patch("coder_eval.sandbox.Sandbox.adopt") as adopt, + patch("coder_eval.sandbox.Sandbox.setup") as setup, + ): + runner.invoke(app, args) + + assert adopt.called is expect_adopt + assert setup.called is not expect_adopt + + +def test_run_evaluation_has_real_defaults_not_typer_sentinels(tmp_path: Path) -> None: + """`run_evaluation` exists because calling the Typer command in-process hands + every unspecified option an `OptionInfo` — and `in_place=None` became truthy, + silently flipping the copy default to in-place.""" + import inspect + + sig = inspect.signature(run_evaluation) + for name in ("work_dir", "workspace", "in_place", "run_dir"): + assert sig.parameters[name].default is None, f"{name} must default to a real None" + assert sig.parameters["preserve"].default is True + + +# -------------------------------------------------------------------------- +# The PATH round trip +# -------------------------------------------------------------------------- + + +def test_the_agents_path_is_persisted_so_a_later_grade_can_restore_it(tmp_path: Path) -> None: + """Without the persisted value a detached grade resolves `run_command` + binaries against ambient PATH and can disagree with the run it grades.""" + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v") + orch.result = EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2026, 1, 1), + final_status=FinalStatus.FAILURE, + iteration_count=0, + ) + orch.sandbox = MagicMock() + orch.agent = MagicMock() + orch.agent.get_sdk_options.return_value = {"env": {"PATH": f"{tmp_path}:/usr/bin"}} + + orch._sync_sandbox_command_path_with_agent() + + assert "command_base_path" in orch.result.environment_info + + +def test_a_restored_path_drops_entries_inside_the_graded_workspace(tmp_path: Path) -> None: + """The restored value is PREPENDED ahead of the host PATH and comes out of the + run's own task.json. An entry inside the agent-writable workspace could + shadow a real tool on the grader's host.""" + workspace = tmp_path / "ws" + (workspace / "bin").mkdir(parents=True) + outside = tmp_path / "toolchain" + outside.mkdir() + + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v") + orch.sandbox = MagicMock() + orch.sandbox.sandbox_dir = workspace + + kept = orch._sanitize_restored_path(f"{workspace / 'bin'}:{outside}:{tmp_path / 'gone'}") + + assert str(outside.resolve()) in kept + assert str(workspace) not in kept, "an entry inside the graded tree must be dropped" + assert "gone" not in kept, "a non-existent entry buys no parity" + + +# -------------------------------------------------------------------------- +# The LiteLLM cost join +# -------------------------------------------------------------------------- + + +def test_the_actual_cost_join_is_skipped_on_a_re_grade(tmp_path: Path) -> None: + """The join keys on a per-Orchestrator nonce the prior turns never carried, + so running it on a re-grade would clobber already-correct per-turn costs.""" + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + prior = EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2026, 1, 1), + final_status=FinalStatus.NOT_GRADED, + iteration_count=0, + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v", prior_result=prior) + orch.result = prior + + with patch("coder_eval.litellm_cost.apply_actual_cost") as apply: + orch._join_litellm_actual_cost() + + apply.assert_not_called() diff --git a/tests/test_early_stop.py b/tests/test_early_stop.py index 5f647392..ff57c63d 100644 --- a/tests/test_early_stop.py +++ b/tests/test_early_stop.py @@ -38,7 +38,7 @@ from coder_eval.agents.claude_code_agent import ClaudeCodeAgent from coder_eval.agents.codex_agent import CodexAgent, _CodexTurnState from coder_eval.agents.registry import AgentRegistry -from coder_eval.cli.plan_command import plan_command +from coder_eval.cli.plan_command import run_plan from coder_eval.config import settings from coder_eval.criteria import CriterionRegistry, init_criteria from coder_eval.criteria.command_executed import CommandExecutedChecker @@ -1041,7 +1041,7 @@ def test_run_surface_variant_inherits_task_threshold_with_stop_early_false(self, assert early_stop_active(by_variant["smoke"]) is True def _run_plan(self, task_file: Path, exp_dir: Path) -> tuple[str, int]: - """Invoke the real plan_command against a minimal single-variant experiment. + """Invoke the real plan body against a minimal single-variant experiment. Returns the concatenated console output and the exit code (0 when plan returned normally). @@ -1058,7 +1058,7 @@ def _run_plan(self, task_file: Path, exp_dir: Path) -> tuple[str, int]: patch("coder_eval.cli.plan_command.console") as mock_console, ): try: - plan_command(task_files=[task_file], experiment=exp_file) + run_plan(task_files=[task_file], experiment=exp_file) except typer.Exit as exc: exit_code = exc.exit_code printed = " ".join(str(call) for call in mock_console.print.call_args_list) diff --git a/tests/test_execute_command.py b/tests/test_execute_command.py index be27c2a8..c0189b54 100644 --- a/tests/test_execute_command.py +++ b/tests/test_execute_command.py @@ -213,7 +213,7 @@ def _option_names(command: str) -> set[str]: } -def test_execute_exposes_run_flags_minus_the_two_refused_ones() -> None: +def test_execute_exposes_run_flags_minus_the_refused_one() -> None: run_opts = _option_names("run") execute_opts = _option_names("execute") @@ -282,10 +282,10 @@ def test_container_defaults_to_grading_when_the_host_sends_no_key() -> None: def test_execute_help_explains_the_refused_flags() -> None: - """The two omissions are documented in the help, not silently absent — a user - who reaches for `--resume` needs to learn why it is refused, not just that it + """The omission is documented in the help, not silently absent — a user who + reaches for `--junit-xml` needs to learn why it is refused, not just that it is unrecognised. (Presence as a real *flag* is covered by the option-set test - above; here we only require the help text to mention them.)""" + above; here we only require the help text to mention it.)""" result = runner.invoke(app, ["execute", "--help"]) assert result.exit_code == 0 for flag in _DELIBERATELY_ABSENT_FROM_EXECUTE: diff --git a/tests/test_execute_evaluate_loop.py b/tests/test_execute_evaluate_loop.py index 96c0b415..cd6f566d 100644 --- a/tests/test_execute_evaluate_loop.py +++ b/tests/test_execute_evaluate_loop.py @@ -14,6 +14,7 @@ import shutil from pathlib import Path from typing import Any +from unittest.mock import AsyncMock, patch import pytest from typer.testing import CliRunner @@ -262,3 +263,98 @@ def test_execute_to_run_resume_emits_no_config_drift_warning(tmp_path: Path) -> result = _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) assert "run config changed" not in result.output + + +def test_run_resume_keeps_the_row_regradeable_when_grading_crashes(tmp_path: Path) -> None: + """A grading crash is not a verdict about the run. Folding the ORIGINAL + ungraded row back keeps the task re-gradeable — writing ERROR over it would + not, since ERROR is "complete" for both commands.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + + with patch( + "coder_eval.orchestration.regrade.regrade_in_place", + new=AsyncMock(side_effect=RuntimeError("checker exploded")), + ): + result = runner.invoke(app, ["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert result.exit_code != 0, "a resume that graded nothing must not report success" + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.NOT_GRADED.value + # The reason is durable, not console-only. It lands in run.json rather than + # task.json: task.json stays the pristine execute record, which is what keeps + # the row re-gradeable below. + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert "checker exploded" in str(summary["task_results"]) + + # And the row really is still re-gradeable. + _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.SUCCESS.value + + +def test_run_resume_reports_a_failing_verdict_and_exits_non_zero(tmp_path: Path) -> None: + """The other resume gate: grading that SUCCEEDS but fails the criteria.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + # Remove the file the criteria read, so the grade legitimately fails. + for proof in run_dir.glob("**/artifacts/**/proof.txt"): + proof.unlink() + + result = runner.invoke(app, ["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert result.exit_code != 0 + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.FAILURE.value + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert summary["tasks_failed"] == 1 + assert summary["tasks_not_graded"] == 0 + + +def test_evaluate_grades_the_directory_named_by_workspace(tmp_path: Path) -> None: + """--workspace exists for a verifier that built its own /app; nothing else + asserted it actually grades that directory rather than the run's artifacts.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + (elsewhere / "proof.txt").write_text("coder-eval-ran-without-a-coder", encoding="utf-8") + # Make the run's own artifacts FAIL, so a pass can only come from --workspace. + for proof in run_dir.glob("**/artifacts/**/proof.txt"): + proof.unlink() + + _invoke(["evaluate", str(_task_dir(run_dir)), "--workspace", str(elsewhere)]) + + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.SUCCESS.value + + +def test_evaluate_refuses_to_re_grade_a_run_that_errored(tmp_path: Path) -> None: + """Grading may only move NOT_GRADED to a verdict. An ERROR / TIMEOUT run is + an execution fact this pass neither repeated nor observed — laundering it + into SUCCESS would report a crashed run as a pass.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + row = _row(task_dir) + row["final_status"] = FinalStatus.TIMEOUT.value + (task_dir / "task.json").write_text(json.dumps(row), encoding="utf-8") + + _invoke(["evaluate", str(task_dir)]) + + assert _row(task_dir)["final_status"] == FinalStatus.TIMEOUT.value + + +def test_grading_the_same_run_twice_reaches_the_same_verdict(tmp_path: Path) -> None: + """Idempotence. A second grade must see the same workspace the first did — + it catches both a pre_run that mutated the tree and a lost sandbox_path.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + + _invoke(["evaluate", str(task_dir)]) + first = _row(task_dir) + _invoke(["evaluate", str(task_dir)]) + second = _row(task_dir) + + assert second["final_status"] == first["final_status"] + assert second["weighted_score"] == first["weighted_score"] + assert second["sandbox_path"] == first["sandbox_path"], "the artifacts pointer must survive a re-grade" + # The pre-grade record is still the ORIGINAL ungraded one, not the first grade's. + assert _row(task_dir, "task.execute.json")["final_status"] == FinalStatus.NOT_GRADED.value diff --git a/tests/test_plan_command.py b/tests/test_plan_command.py index 16a8d663..f8796fb9 100644 --- a/tests/test_plan_command.py +++ b/tests/test_plan_command.py @@ -6,7 +6,7 @@ import pytest import typer -from coder_eval.cli.plan_command import plan_command +from coder_eval.cli.plan_command import run_plan from coder_eval.models import ( AgentConfig, ExperimentDefinition, @@ -73,7 +73,7 @@ def test_plan_shows_na_when_agent_is_none(self, tmp_path: Path) -> None: patch(f"{_EXP}.resolve_task_for_variant", return_value=(resolved_task, {}, 1)), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) printed = [str(call) for call in mock_console.print.call_args_list] agent_lines = [p for p in printed if "Agent:" in p] @@ -98,7 +98,7 @@ def test_plan_shows_agent_type_when_present(self, tmp_path: Path) -> None: patch(f"{_EXP}.resolve_task_for_variant", return_value=(resolved_task, {}, 1)), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) printed = [str(call) for call in mock_console.print.call_args_list] agent_lines = [p for p in printed if "Agent:" in p] @@ -124,7 +124,7 @@ def test_plan_shows_deferred_when_agent_type_is_none(self, tmp_path: Path) -> No patch(f"{_EXP}.resolve_task_for_variant", return_value=(resolved_task, {}, 1)), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) printed = [str(call) for call in mock_console.print.call_args_list] deferred_lines = [p for p in printed if "deferred" in p] @@ -155,7 +155,7 @@ def test_plan_with_experiment_flag_shows_experiment_info(self, tmp_path: Path) - patch(f"{_EXP}.DEFAULT_EXPERIMENT_PATH", exp_file), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file], experiment=exp_file) + run_plan(task_files=[task_file], experiment=exp_file) printed = " ".join(str(call) for call in mock_console.print.call_args_list) assert "test-exp" in printed @@ -185,7 +185,7 @@ def test_plan_with_experiment_shows_resolved_agent_per_variant(self, tmp_path: P patch(f"{_EXP}.DEFAULT_EXPERIMENT_PATH", exp_file), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file], experiment=exp_file) + run_plan(task_files=[task_file], experiment=exp_file) printed = " ".join(str(call) for call in mock_console.print.call_args_list) assert "sonnet-4" in printed @@ -211,7 +211,7 @@ def test_plan_with_default_experiment(self, tmp_path: Path) -> None: patch(f"{_EXP}.resolve_task_for_variant", return_value=(resolved_task, {}, 1)), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) printed = " ".join(str(call) for call in mock_console.print.call_args_list) assert "test-exp" in printed @@ -231,7 +231,7 @@ def test_plan_warns_when_task_timeout_cannot_extend_single_iteration(self, tmp_p patch(f"{_EXP}.resolve_task_for_variant", return_value=(resolved_task, {}, 1)), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) printed = " ".join(str(call) for call in mock_console.print.call_args_list) assert "A larger task_timeout cannot extend the agent's single iteration" in printed @@ -254,7 +254,7 @@ def test_plan_exits_when_default_experiment_missing(self, tmp_path: Path) -> Non patch("coder_eval.cli.plan_command.console") as mock_console, ): with pytest.raises(typer.Exit) as exc_info: - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) assert exc_info.value.exit_code == 1 @@ -280,7 +280,7 @@ def test_plan_reports_invalid_task(self, tmp_path: Path) -> None: patch("coder_eval.cli.plan_command.console") as mock_console, ): with pytest.raises(typer.Exit) as exc_info: - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) assert exc_info.value.exit_code == 1 @@ -305,7 +305,7 @@ def test_plan_exits_on_explicit_experiment_load_failure(self, tmp_path: Path) -> patch("coder_eval.cli.plan_command.console") as mock_console, ): with pytest.raises(typer.Exit) as exc_info: - plan_command(task_files=[task_file], experiment=exp_file) + run_plan(task_files=[task_file], experiment=exp_file) assert exc_info.value.exit_code == 1 diff --git a/tests/test_regrade.py b/tests/test_regrade.py index 668046c8..6663089b 100644 --- a/tests/test_regrade.py +++ b/tests/test_regrade.py @@ -27,8 +27,6 @@ parse_agent_config, ) from coder_eval.orchestration.regrade import ( - PRE_GRADE_JSON, - TASK_JSON, RegradeError, back_up_pre_grade_record, default_workspace, @@ -36,7 +34,7 @@ task_from_prior, verify_reference_unchanged, ) -from coder_eval.path_utils import digest_tree +from coder_eval.path_utils import PRE_GRADE_JSON_FILENAME, TASK_JSON_FILENAME, digest_tree def _task(*, reference: dict[str, str] | None = None, command: str | None = None) -> TaskDefinition: @@ -80,7 +78,7 @@ def test_missing_task_json_is_a_regrade_error(tmp_path: Path) -> None: def test_unparseable_task_json_is_a_regrade_error(tmp_path: Path) -> None: - (tmp_path / TASK_JSON).write_text("{not json", encoding="utf-8") + (tmp_path / TASK_JSON_FILENAME).write_text("{not json", encoding="utf-8") with pytest.raises(RegradeError, match="not a readable EvaluationResult"): load_prior_result(tmp_path) @@ -235,25 +233,64 @@ def test_a_task_with_no_reference_is_not_checked(tmp_path: Path) -> None: def test_the_pre_grade_record_is_written_once(tmp_path: Path) -> None: """A second grade must not overwrite the ORIGINAL execute record with an already-graded one — that is the only evidence the run was ungraded.""" - (tmp_path / TASK_JSON).write_text('{"round": 1}', encoding="utf-8") + (tmp_path / TASK_JSON_FILENAME).write_text('{"round": 1}', encoding="utf-8") back_up_pre_grade_record(tmp_path) - (tmp_path / TASK_JSON).write_text('{"round": 2}', encoding="utf-8") + (tmp_path / TASK_JSON_FILENAME).write_text('{"round": 2}', encoding="utf-8") back_up_pre_grade_record(tmp_path) - assert json.loads((tmp_path / PRE_GRADE_JSON).read_text(encoding="utf-8")) == {"round": 1} + assert json.loads((tmp_path / PRE_GRADE_JSON_FILENAME).read_text(encoding="utf-8")) == {"round": 1} def test_backup_is_a_no_op_with_nothing_to_back_up(tmp_path: Path) -> None: back_up_pre_grade_record(tmp_path) - assert not (tmp_path / PRE_GRADE_JSON).exists() + assert not (tmp_path / PRE_GRADE_JSON_FILENAME).exists() def test_a_failed_backup_never_fails_the_grade(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """The audit copy is a convenience; the verdict is the deliverable.""" - (tmp_path / TASK_JSON).write_text("{}", encoding="utf-8") + (tmp_path / TASK_JSON_FILENAME).write_text("{}", encoding="utf-8") def _boom(*_args: object, **_kwargs: object) -> None: raise OSError("read-only file system") monkeypatch.setattr(Path, "write_text", _boom) back_up_pre_grade_record(tmp_path) # must not raise + + +# -------------------------------------------------------------------------- +# Dataset-row ids contain "/" +# -------------------------------------------------------------------------- + + +def test_a_dataset_row_workspace_resolves_by_task_id_not_by_child_count(tmp_path: Path) -> None: + """Preservation writes artifacts/, and a dataset row's task_id is + "/". "The single child of artifacts/" therefore resolves to + artifacts/ — one level too high — and every path-relative criterion + then fails as a locating artifact rather than as a verdict.""" + workspace = tmp_path / "artifacts" / "suite" / "row-1" + workspace.mkdir(parents=True) + + resolved = default_workspace(tmp_path, _result(task_id="suite/row-1")) + + assert resolved == workspace + + +def test_an_ambiguous_artifacts_dir_refuses_rather_than_guessing(tmp_path: Path) -> None: + artifacts = tmp_path / "artifacts" + (artifacts / "a").mkdir(parents=True) + (artifacts / "b").mkdir() + + with pytest.raises(RegradeError, match="Pass --workspace"): + default_workspace(tmp_path, _result(task_id="neither")) + + +def test_a_sandbox_path_outside_the_run_dir_is_refused(tmp_path: Path) -> None: + """`sandbox_path` is an unvalidated absolute path out of the run's own + task.json, and criteria execute with cwd there and may mutate it.""" + outside = tmp_path / "elsewhere" + outside.mkdir() + run_dir = tmp_path / "run" + run_dir.mkdir() + + with pytest.raises(RegradeError, match="outside the run directory"): + default_workspace(run_dir, _result(sandbox_path=str(outside))) diff --git a/tests/test_seed_from_prior_result.py b/tests/test_seed_from_prior_result.py index 319815d3..10b602cc 100644 --- a/tests/test_seed_from_prior_result.py +++ b/tests/test_seed_from_prior_result.py @@ -106,7 +106,7 @@ def _prior() -> EvaluationResult: pre_run_results=[PostRunResult(command="prior-pre", exit_code=0)], post_run_results=[PostRunResult(command="prior-post", exit_code=0)], sandbox_path="/prior/workspace", - environment_info={"installed_tools": "prior"}, + environment_info={"installed_tools": "prior", "coder_eval": "1.0.0-run"}, early_stop=EarlyStopInfo( reason=EarlyStopReason.CRITERION_FAILED, deciding_criterion_type="skill_triggered", @@ -137,7 +137,7 @@ def _seeded(tmp_path: Path) -> tuple[Orchestrator, EvaluationResult]: started_at=datetime(2030, 1, 1, 0, 0, 0), final_status=FinalStatus.FAILURE, iteration_count=0, - environment_info={"installed_tools": "grader"}, + environment_info={"installed_tools": "grader", "coder_eval": "9.9.9-grader"}, ) orch._seed_from_prior_result() assert orch.result is not None @@ -182,7 +182,13 @@ def test_grader_environment_is_kept_beside_the_run_s_not_over_it(tmp_path: Path) # The run's own capture wins: a report showing the grader's tool versions as # the run's is worse than one showing neither. assert orch.result.environment_info["installed_tools"] == "prior" - assert orch.result.environment_info["graded_by"] == {"installed_tools": "grader"} + # The grader is recorded as FLAT scalars, and only where it differs. Nesting + # a whole env capture here renders as a Python dict repr in the HTML report + # and violates the evalboard's declared value type. + assert orch.result.environment_info["graded_by_coder_eval"] == "9.9.9-grader" + assert all(not isinstance(v, dict) or k == "installed_tools" for k, v in orch.result.environment_info.items()), ( + "environment_info is consumed as a flat map" + ) def test_the_evaluate_only_path_selects_the_same_gate_as_the_agent_path(tmp_path: Path) -> None: diff --git a/tests/test_ungraded_reporting.py b/tests/test_ungraded_reporting.py new file mode 100644 index 00000000..db2b9933 --- /dev/null +++ b/tests/test_ungraded_reporting.py @@ -0,0 +1,254 @@ +"""How an ungraded row renders on every reporting surface. + +`coder-eval execute` leaves rows `NOT_GRADED`, and each generator had to learn a +fourth category. Only the HTML badge got an assertion when that landed, so the +JUnit `` element, the Markdown "Not Graded" bullets, and — most +importantly — the switched pass-rate DENOMINATOR were all shipped untested. The +denominator is the "gate that turns a gap into a score" shape: a printed rate +whose divisor changed, with nothing tripping it. + +`VariantAggregate` is here too, mirroring the four `RunSummary` cases in +`test_execute_command.py`: the two models now carry the same formula, and only +one of them was tested. +""" + +from __future__ import annotations + +from datetime import datetime +from pathlib import Path +from typing import Any + +import pytest + +from coder_eval.models import FinalStatus, RunSummary, VariantAggregate, VariantResult +from coder_eval.orchestration.experiment import _mean_graded_score, _pick_worst_status + + +def _summary(**kwargs: object) -> RunSummary: + base: dict[str, object] = { + "run_id": "r", + "start_time": datetime(2026, 1, 1), + "end_time": datetime(2026, 1, 1, 0, 1), + "total_duration_seconds": 60.0, + "tasks_run": 2, + "tasks_succeeded": 1, + "tasks_failed": 0, + "tasks_error": 0, + "tasks_not_graded": 1, + "task_results": [], + "framework_version": "test", + } + base.update(kwargs) + return RunSummary(**base) # type: ignore[arg-type] + + +def _aggregate(**kwargs: object) -> VariantAggregate: + base: dict[str, object] = { + "variant_id": "v", + "tasks_run": 2, + "tasks_succeeded": 1, + "tasks_failed": 0, + "tasks_error": 0, + "tasks_not_graded": 1, + "average_score": 1.0, + "average_duration": 1.0, + } + base.update(kwargs) + return VariantAggregate(**base) # type: ignore[arg-type] + + +# -------------------------------------------------------------------------- +# VariantAggregate — the twin of the RunSummary cases in test_execute_command +# -------------------------------------------------------------------------- + + +def test_variant_aggregate_counts_the_ungraded_bucket_in_its_invariant() -> None: + with pytest.raises(ValueError, match="Task count invariant violated"): + _aggregate(tasks_run=3) # 1 + 0 + 0 + 1 != 3 + + +def test_variant_aggregate_pass_rate_divides_by_graded_not_run() -> None: + """One pass out of one GRADED task is 100%, even beside an ungraded one.""" + assert _aggregate().tasks_graded == 1 + assert _aggregate().pass_rate == 1.0 + + +def test_variant_aggregate_has_no_pass_rate_when_nothing_was_graded() -> None: + agg = _aggregate(tasks_run=2, tasks_succeeded=0, tasks_not_graded=2) + assert agg.pass_rate is None, "0/0 is unknown, not 0%" + + +def test_variant_aggregate_serializes_its_denominator() -> None: + """A consumer that cannot read `tasks_graded` re-derives the rate and drifts.""" + import json + + assert json.loads(_aggregate().model_dump_json())["tasks_graded"] == 1 + + +def test_mean_graded_score_ignores_ungraded_rows() -> None: + def _vr(score: float | None, status: FinalStatus) -> VariantResult: + return VariantResult( + variant_id="v", task_id="t", weighted_score=score, final_status=status, duration_seconds=1.0 + ) + + rows = [_vr(1.0, FinalStatus.SUCCESS), _vr(None, FinalStatus.NOT_GRADED), _vr(None, FinalStatus.NOT_GRADED)] + assert _mean_graded_score(rows) == 1.0 + # Nothing graded -> no mean at all. 0.0 would read as "measured and scored zero". + assert _mean_graded_score(rows[1:]) is None + + +def test_ungraded_loses_to_every_real_outcome_when_picking_the_worst_status() -> None: + """`_pick_worst_status` reports the replicate set's worst outcome. An ungraded + replicate is not an outcome, so it must never mask a real one.""" + assert _pick_worst_status([FinalStatus.NOT_GRADED, FinalStatus.SUCCESS]) is FinalStatus.SUCCESS + assert _pick_worst_status([FinalStatus.NOT_GRADED, FinalStatus.FAILURE]) is FinalStatus.FAILURE + assert _pick_worst_status([FinalStatus.NOT_GRADED]) is FinalStatus.NOT_GRADED + + +# -------------------------------------------------------------------------- +# Markdown +# -------------------------------------------------------------------------- + + +def test_markdown_pass_rate_uses_the_graded_denominator() -> None: + from coder_eval.reports import _pass_rate_lines + + text = "\n".join(_pass_rate_lines(_summary())) + + assert "1/1" in text, "the denominator is tasks_graded, not tasks_run" + assert "1/2" not in text + + +def test_markdown_reports_no_rate_at_all_for_a_fully_ungraded_run() -> None: + from coder_eval.reports import _pass_rate_lines + + text = "\n".join(_pass_rate_lines(_summary(tasks_succeeded=0, tasks_not_graded=2))) + + assert "n/a" in text + assert "0.0%" not in text, "a run that was never measured has no rate, not a 0% one" + + +def test_markdown_reports_no_rate_change_for_an_ordinary_graded_run() -> None: + """The regression guard: adding the fourth bucket must not alter any surface + of a run that has none.""" + from coder_eval.reports import _pass_rate_lines + + text = "\n".join(_pass_rate_lines(_summary(tasks_run=2, tasks_succeeded=1, tasks_failed=1, tasks_not_graded=0))) + + assert "1/2" in text + assert "n/a" not in text + + +# -------------------------------------------------------------------------- +# JUnit +# -------------------------------------------------------------------------- + + +def _junit_for(status: FinalStatus, tmp_path: Path) -> Any: + import json + + # defusedxml on the test side, matching tests/test_reports_junit.py. + from defusedxml.ElementTree import parse as parse_xml + + from coder_eval.reports_junit import write_junit_xml + + run_dir = tmp_path / "run" + task_dir = run_dir / "default" / "t" / "00" + task_dir.mkdir(parents=True) + row = { + "task_id": "t", + "task_description": "d", + "variant_id": "default", + "agent_type": "claude-code", + "started_at": "2026-01-01T00:00:00", + "final_status": status.value, + "iteration_count": 1, + "duration_seconds": 1.0, + } + (task_dir / "task.json").write_text(json.dumps(row), encoding="utf-8") + (run_dir / "run.json").write_text( + json.dumps( + { + "run_id": "r", + "start_time": "2026-01-01T00:00:00", + "end_time": "2026-01-01T00:01:00", + "total_duration_seconds": 60.0, + "tasks_run": 1, + "tasks_succeeded": 1 if status is FinalStatus.SUCCESS else 0, + "tasks_failed": 0, + "tasks_error": 0, + "tasks_not_graded": 1 if status is FinalStatus.NOT_GRADED else 0, + "task_results": [{"task_id": "t", "variant_id": "default", "status": status.value, "duration": 1.0}], + "framework_version": "test", + } + ), + encoding="utf-8", + ) + written = write_junit_xml(run_dir, tmp_path / "junit.xml") + return parse_xml(written).getroot() + + +def test_junit_marks_an_ungraded_row_skipped_not_failed(tmp_path: Path) -> None: + root = _junit_for(FinalStatus.NOT_GRADED, tmp_path) + + skipped = root.findall(".//testcase/skipped") + assert len(skipped) == 1, "an ungraded row is not a verdict, so it is " + assert "not graded" in (skipped[0].get("message") or "") + assert not root.findall(".//testcase/failure"), "an ungraded row must not read as a failure" + + +def test_junit_counts_are_derived_from_the_children(tmp_path: Path) -> None: + root = _junit_for(FinalStatus.NOT_GRADED, tmp_path) + suite = root.find(".//testsuite") + assert suite is not None + assert suite.get("skipped") == "1" + assert suite.get("failures") == "0" + assert suite.get("errors") == "0" + + +def test_junit_still_passes_an_ordinary_graded_row(tmp_path: Path) -> None: + root = _junit_for(FinalStatus.SUCCESS, tmp_path) + assert not root.findall(".//testcase/skipped") + assert not root.findall(".//testcase/failure") + + +# -------------------------------------------------------------------------- +# The end-of-run console summary +# -------------------------------------------------------------------------- + + +def _summary_output(summary: RunSummary, tmp_path: Path) -> str: + from coder_eval.cli.console import console + from coder_eval.cli.run_helpers import print_execution_summary + + with console.capture() as captured: + print_execution_summary(tmp_path, summary) + return captured.get() + + +def test_summary_reports_an_ungraded_run_as_executed_not_failed(tmp_path: Path) -> None: + text = _summary_output(_summary(tasks_succeeded=0, tasks_not_graded=2), tmp_path) + assert "2/2 executed, not graded" in text + assert "0/0 succeeded" not in text, "a fully ungraded run has no succeeded ratio to report" + + +def test_summary_points_at_the_grading_form_that_keeps_the_trajectory(tmp_path: Path) -> None: + """`evaluate ` grades a bare directory with NO + trajectory, so command_executed / skill_triggered / trajectory judges score + differently from what `run` would have produced.""" + text = _summary_output(_summary(tasks_succeeded=0, tasks_not_graded=2), tmp_path) + assert "--resume" in text or "" in text + assert "evaluate " not in text + + +def test_summary_still_reports_a_ratio_for_an_empty_run(tmp_path: Path) -> None: + """Both counters are falsy for tasks_run == 0; independent `if`s would print + no Results line at all, where it previously printed 0/0.""" + empty = _summary(tasks_run=0, tasks_succeeded=0, tasks_not_graded=0) + assert "0/0 succeeded" in _summary_output(empty, tmp_path) + + +def test_summary_is_unchanged_for_an_ordinary_graded_run(tmp_path: Path) -> None: + text = _summary_output(_summary(tasks_run=2, tasks_succeeded=1, tasks_failed=1, tasks_not_graded=0), tmp_path) + assert "1/2 succeeded" in text + assert "not graded" not in text From e9dfb947ad9aed0ccdd41bb666d2cb9c05643e9d Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Thu, 3 Sep 2026 16:38:32 -0700 Subject: [PATCH 06/11] test: fix two CI-only failures in the detached-grading tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both are test bugs, not product bugs, and both are the same class: an assertion that passes on the developer's machine and only on the developer's machine. `_sanitize_restored_path` splits on `os.pathsep`; its test built the input with a hardcoded ":". On Windows that parses as ONE non-existent entry, so the sanitizer returns "" and every assertion below it passes vacuously — the test was asserting nothing on the platform it failed on. Rich splits an `--option` token across several style spans (`--junit-xml` renders as `-` + `-junit` + `-xml`, each with its own escape), and it styles whenever it believes it is writing to a terminal — which includes GitHub Actions. So a bare substring check over `result.output` is green locally and red only in CI. Strip ANSI first, following the helper and the comment already in tests/test_cli_type_flag.py. Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_detached_grading_guards.py | 7 ++++++- tests/test_execute_command.py | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/test_detached_grading_guards.py b/tests/test_detached_grading_guards.py index 2ff234cf..d0f66b18 100644 --- a/tests/test_detached_grading_guards.py +++ b/tests/test_detached_grading_guards.py @@ -7,6 +7,7 @@ from __future__ import annotations +import os from datetime import datetime from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch @@ -221,7 +222,11 @@ def test_a_restored_path_drops_entries_inside_the_graded_workspace(tmp_path: Pat orch.sandbox = MagicMock() orch.sandbox.sandbox_dir = workspace - kept = orch._sanitize_restored_path(f"{workspace / 'bin'}:{outside}:{tmp_path / 'gone'}") + # os.pathsep, not a hardcoded ":" — the separator is ";" on Windows, where a + # colon-joined value parses as one (non-existent) entry and every assertion + # below passes vacuously against an empty result. + recorded = os.pathsep.join([str(workspace / "bin"), str(outside), str(tmp_path / "gone")]) + kept = orch._sanitize_restored_path(recorded) assert str(outside.resolve()) in kept assert str(workspace) not in kept, "an entry inside the graded tree must be dropped" diff --git a/tests/test_execute_command.py b/tests/test_execute_command.py index c0189b54..e7686c6b 100644 --- a/tests/test_execute_command.py +++ b/tests/test_execute_command.py @@ -16,6 +16,7 @@ from __future__ import annotations import json +import re from pathlib import Path from typing import Any from unittest.mock import patch @@ -29,6 +30,19 @@ runner = CliRunner() +# Rich styles each `--option` token in help text, and it splits the token across +# several style spans (`--junit-xml` renders as `-` + `-junit` + `-xml`, each with +# its own escape sequence). Styling is ON whenever rich thinks it is writing to a +# terminal — which includes GitHub Actions, so a bare substring check over +# `result.output` passes locally and fails only in CI. Same helper, same reason, +# as tests/test_cli_type_flag.py. +_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") + + +def _strip_ansi(s: str) -> str: + return _ANSI_RE.sub("", s) + + # The agentless smoke task: no agent, no model call, and a pre_run that writes a # file its criteria read back. Executing it must still write that file (proving # the run really happened) while scoring nothing. @@ -288,5 +302,6 @@ def test_execute_help_explains_the_refused_flags() -> None: above; here we only require the help text to mention it.)""" result = runner.invoke(app, ["execute", "--help"]) assert result.exit_code == 0 + output = _strip_ansi(result.output) for flag in _DELIBERATELY_ABSENT_FROM_EXECUTE: - assert flag in result.output, f"execute's help should explain why {flag} is unavailable" + assert flag in output, f"execute's help should explain why {flag} is unavailable" From 7d5d55d6bc092603e5f0b573f197ea1aca2970c6 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Fri, 4 Sep 2026 10:36:36 -0700 Subject: [PATCH 07/11] fix(execute): close the verdict-changing and trust-boundary defects in detached grading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the PR #154 review. The `execute`/`evaluate` split shipped with the right shape but several ways to produce a plausible number that is wrong. Verdict-changing: * `grading_sandbox_config` rewrote `driver: docker` -> `tempdir` unconditionally, so a container task's criteria ran on the grading host — scoring FAILURE for a trajectory `run` scored 1.0, running `rm -rf /verifier` unsandboxed, and neutralizing `Sandbox.adopt`'s own docker refusal. Now refused unless `--allow-host-grading`; an opted-in row is stamped `graded_on_host`. * `max_turns_exhausted` and `_check_run_limits` sat after the grading early return, so `execute` exited 0 where `run` exited 1 for identical agent output — and `_seed_from_prior_result` cannot restore a fact never captured. * Experiment aggregation filtered on `weighted_score is not None`, dropping ERROR/BUILD_FAILED rows from BOTH sides: an infrastructure-failure night scored higher than a clean one. Only `ungraded` leaves both sides now. * `verify_reference_unchanged` compared a staged-copy digest against the raw source, so any `.git`-carrying reference reported a permanent false mismatch. * A grading crash left ERROR on disk (`_finalize_result` writes before returning), making the row permanently un-regradeable and leaving run.json disagreeing with task.json. Trust boundary — a run directory is a shareable artifact: * A recorded config carrying shell is refused unless `--allow-recorded-commands` (hooks excluded on the in-place path, where they do not run). * `artifacts / prior.task_id` is containment-checked like its `sandbox_path` sibling. * `write_text_atomic` opens `O_EXCL|O_NOFOLLOW`, closing the `task.json.tmp` symlink primitive that bypassed the write-back's own guard. * `_sanitize_restored_path` drops relative entries and anything in the run dir. Reporting and evalboard: * `SuiteRollup.pass_rate` is `float | None` with a serialized `rows_graded`; ungraded rows leave `failed_samples`. * Telemetry omits `Score` rather than laundering `None` into a real-looking 0.0. * A detached grade records `graded_by_api_routing` instead of overwriting the run's. * `reports_stats` drops only the score, not the whole row — duration, tokens and turns are facts about the run, not verdicts. * The evalboard's ungraded fields had no readers, so a 12-task `execute` run rendered a red `0% - 0 / 12`. Swept every rate surface and gave `StatusCategory` an `assertNever` guard, since widening the union produced no compiler error anywhere and that is how `lib/overview.ts` was missed. New lint rules, each traceable to one of the above: CE049 (no `score or 0.0`), CE050 (no untyped `getattr` probe over a discriminated union), CE051 (no silent sandbox-driver rewrite). Adding fires-on-violation tests for CE047/CE048 also surfaced the scoping bug CE047 warns about: its `[/\\]src[/\\]` regex put every repo-relative path out of scope, so such a test would have passed vacuously. Two cheap extractions (`_terminal_status`, `_apply_resume`) plus `_fold_replicates` undo the complexity the grading switch added: `aggregate_results` F(54) -> E(36), below its pre-PR F(48). Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 4 +- docs/REPORT_SCHEMA.md | 11 +- docs/USER_GUIDE.md | 19 +- evalboard/app/_overview/window-summary.tsx | 4 +- evalboard/app/page.tsx | 17 +- .../path-to-ga/__tests__/task-table.test.tsx | 1 + .../runs/[id]/__tests__/task-grid.test.tsx | 8 +- .../[id]/__tests__/ungraded-golden.test.ts | 148 ++++++ evalboard/app/runs/[id]/run-view.tsx | 52 ++- evalboard/app/runs/[id]/task-grid.tsx | 26 +- evalboard/app/scribe/run-table.tsx | 8 +- evalboard/lib/__tests__/overview.test.ts | 2 + evalboard/lib/__tests__/status-parity.test.ts | 77 ++++ evalboard/lib/overview.ts | 50 ++- evalboard/lib/pills.tsx | 33 +- evalboard/lib/runs.ts | 13 + evalboard/lib/status.ts | 40 +- plugins/coder-eval/skills/analyze/SKILL.md | 11 + pyproject.toml | 3 + src/coder_eval/cli/__init__.py | 2 +- src/coder_eval/cli/aggregate_command.py | 14 +- src/coder_eval/cli/evaluate_command.py | 140 ++++-- src/coder_eval/cli/evaluate_target.py | 22 + src/coder_eval/cli/execute_command.py | 8 +- src/coder_eval/cli/run_command.py | 156 +++++-- .../cli/run_task_internal_command.py | 6 +- src/coder_eval/models/results.py | 25 +- src/coder_eval/models/tasks.py | 5 +- src/coder_eval/orchestration/experiment.py | 117 +++-- src/coder_eval/orchestration/regrade.py | 226 +++++++++- src/coder_eval/orchestrator.py | 187 +++++--- src/coder_eval/path_utils.py | 26 +- src/coder_eval/reports.py | 29 +- src/coder_eval/reports_experiment.py | 14 +- src/coder_eval/reports_html.py | 11 + src/coder_eval/reports_stats.py | 25 +- .../rules/ce047_env_info_key_round_trip.py | 6 +- tests/lint/rules/ce049_no_score_or_zero.py | 70 +++ .../rules/ce050_no_union_getattr_probe.py | 132 ++++++ tests/lint/rules/ce051_no_driver_override.py | 94 ++++ tests/lint/runner.py | 6 + tests/test_custom_lint.py | 156 +++++++ tests/test_detached_grading_boundaries.py | 423 ++++++++++++++++++ tests/test_detached_grading_guards.py | 39 +- tests/test_execute_command.py | 1 + tests/test_execute_evaluate_loop.py | 68 +++ tests/test_experiment_runner.py | 81 ++++ tests/test_regrade.py | 33 +- tests/test_reports_html.py | 76 ++++ tests/test_route_seam_exhaustiveness.py | 10 +- tests/test_suite_rollup.py | 79 +++- 51 files changed, 2537 insertions(+), 277 deletions(-) create mode 100644 evalboard/app/runs/[id]/__tests__/ungraded-golden.test.ts create mode 100644 evalboard/lib/__tests__/status-parity.test.ts create mode 100644 tests/lint/rules/ce049_no_score_or_zero.py create mode 100644 tests/lint/rules/ce050_no_union_getattr_probe.py create mode 100644 tests/lint/rules/ce051_no_driver_override.py create mode 100644 tests/test_detached_grading_boundaries.py diff --git a/CLAUDE.md b/CLAUDE.md index 71e487f3..1474465a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -151,7 +151,7 @@ action.yml # Published composite GitHub Action (coder-ev - **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex and Antigravity (both run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced), and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. Full table + rationale: docs/agents/HARNESS_PARITY.md. - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly. - **Execute vs. run (the grading switch)**: `coder-eval execute` is `coder-eval run` with grading removed — the agent runs and the full trajectory is captured, but no criterion is checked, `weighted_score` is `None` (never `0.0`, which would be indistinguishable from "graded and scored zero"), and the row finalizes as **`FinalStatus.NOT_GRADED`**, whose `category` is a **fourth** bucket, `"ungraded"`. Ungraded rows leave BOTH sides of every rate: `RunSummary.pass_rate` / `error_share` and `VariantAggregate.pass_rate` divide by `tasks_graded` (`tasks_run - tasks_not_graded`), and `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter. **Only SUCCESS/FAILURE collapse into it** — `ERROR`, `TIMEOUT`, `BUILD_FAILED`, `MAX_TURNS_EXHAUSTED` and the budget stops are facts about the *run*, not about grading, and still apply (so `execute` still exits non-zero on a crash). The switch is `BatchRunConfig.grade` → `Orchestrator(grade=...)` → the **four** grading call sites (single-shot, evaluate-only, the simulation dialog check, and post-failure diagnostics); it crosses the docker boundary in `context.json` (defaulting to `True` in-container, so a host predating `execute` keeps grading). It is **deliberately not a task-config field** — no 5-layer merge, no `-D` path — because a task YAML must never declare itself ungraded; only the invoking command decides. `run` and `execute` share one body (`run_command.run_pipeline`) and differ solely in that flag, so there is no third code path. Two things are refused rather than degraded: `--junit-xml` (a report of verdicts, and there are none — though `reports_junit` still emits `` for an ungraded row it encounters) and simulation tasks (their turn-continuation logic reads criteria results, so an ungraded dialog would silently change its own stopping behavior). `stop_early:` blocks are inert under `execute` for the same reason the kill switch exists: the full trajectory is the deliverable. Motivating consumer: an external harness (Harbor / Terminal-Bench 2.0) that builds its own container, calls coder-eval as the agent, and grades with its own tests. -- **Detached grading (`evaluate` over a run dir) + `Sandbox.adopt`**: `coder-eval evaluate` takes two shapes, told apart by a **pure** resolver (`cli/evaluate_target.py`) on one probe — a target holding `task.json` is a run directory. Run-dir mode rebuilds the task from the run's own `task_config.resolved`, **not** by re-loading the YAML: `resolved` is post-merge, so variant overrides / `-D` / dataset expansion are already baked in and re-loading the source would silently grade a *different* task (fallback to `source_file` only when `resolved` no longer validates, and loudly). It seeds the fresh result from the prior one via `Orchestrator(prior_result=...)` → `_seed_from_prior_result`, which carries the trajectory (every derived figure — tokens, cost, `command_stats`, `model_used` — recomputes from `iterations`), `iteration_count`, execution facts, and **`early_stop` — load-bearing, because gate selection is FIRED-ONLY**: dropping it re-grades a truncated trajectory under the full-run strict-AND gate and can flip the verdict. Carrying it is only half the fix — **both** grading paths select the gate through the single `Orchestrator._select_gate()`; the evaluate-only branch a detached grade actually takes originally called `all_criteria_passed` inline, so the seeded field was written and never read. `tests/test_seed_from_prior_result.py` partitions every `EvaluationResult` field as CARRIED or RECOMPUTED and fails closed on a new one, and asserts the two `_select_gate()` call sites. A prior status that `FinalStatus.is_execution_fact` (TIMEOUT / ERROR / BUILD_FAILED / the budget stops) is **preserved**, never overwritten: grading may only move `NOT_GRADED` to SUCCESS/FAILURE, since it neither repeated nor observed the agent phase. Grader-host `environment_info` is preserved under a `graded_by` sub-dict rather than overwriting the run's. Two other parity fixes: `command_base_path` is now persisted into `environment_info` by `_sync_sandbox_command_path_with_agent` and restored in the evaluate-only branch (closing the PATH gap that method's docstring already named), and `_join_litellm_actual_cost` **skips** when `prior_result` is set (its join keys on a per-Orchestrator nonce the prior turns never carried, so it would clobber already-correct costs). The verdict is written back into the run's `task.json`, with the pre-grade record kept as `task.execute.json` — that in-place write is what makes plain `coder-eval aggregate ` rebuild a graded `run.json` with **zero** new code. **`Sandbox.adopt(workspace)`** is the grade-in-place primitive: it reuses `setup`'s adoption half but skips every *materializing* step (`_setup_template`, `_generate_cli_recorders`, venv/package installs, the destructive `$HOME` remediation), running only non-mutating derivation (mock-dir `+x`, venv *discovery*, plugin-tools pin); `_cleanup_on_exit` stays False so an adopted tree is never moved or deleted, and `Sandbox.was_adopted` is set — the Orchestrator reads it to SKIP the `pre_run`/`post_run` hooks (`run()` calls them unconditionally with `cwd = sandbox_dir`, and several in-tree tasks stage fixtures there with `cp -a /app/[!.]* "$PWD/"`, which would overwrite the agent's deliverables before the criteria read them) and to KEEP `sandbox_path` in the `PreservationMode.NONE` cleanup arm (an adopted tree survives cleanup, so the path is not stale). The hooks' recorded results are carried from the prior run instead. In-place is **more correct**, not merely faster: `_setup_template` filters the copy through `_should_ignore_template_file`, which drops `node_modules` / `dist` / `build` / `.venv` / `.git`, so on the copy path a criterion like `test -f dist/bundle.js` fails as a *copying artifact* rather than as a verdict (verified: 0.00 "does not exist" on copy vs 1.00 in place). Defaults: in-place for a run dir, copy for a bare work dir (criteria can mutate it and it is the user's own tree); `--in-place`/`--copy` override. `adopt` hard-errors on `driver: docker` (a container workspace is unreachable from the host) and a re-grade refuses on a `reference_digest` mismatch — the digest is persisted into `environment_info` at staging time by `_stage_reference` (it shipped once as a read with no writer anywhere, so the guard was dead code), and `verify_reference_unchanged` now takes the task file it resolves against and RAISES on a vanished or unresolvable reference instead of returning silently. NOTE the Typer command is a thin wrapper over `run_evaluation(...)`, which has real Python defaults — calling a Typer command function in-process hands unspecified options an `OptionInfo` sentinel, which silently made `in_place=None` truthy. +- **Detached grading (`evaluate` over a run dir) + `Sandbox.adopt`**: `coder-eval evaluate` takes two shapes, told apart by a **pure** resolver (`cli/evaluate_target.py`) on one probe — a target holding `task.json` is a run directory. Run-dir mode rebuilds the task from the run's own `task_config.resolved`, **not** by re-loading the YAML: `resolved` is post-merge, so variant overrides / `-D` / dataset expansion are already baked in and re-loading the source would silently grade a *different* task (fallback to `source_file` only when `resolved` no longer validates, and loudly). It seeds the fresh result from the prior one via `Orchestrator(prior_result=...)` → `_seed_from_prior_result`, which carries the trajectory (every derived figure — tokens, cost, `command_stats`, `model_used` — recomputes from `iterations`), `iteration_count`, execution facts, and **`early_stop` — load-bearing, because gate selection is FIRED-ONLY**: dropping it re-grades a truncated trajectory under the full-run strict-AND gate and can flip the verdict. Carrying it is only half the fix — **both** grading paths select the gate through the single `Orchestrator._select_gate()`; the evaluate-only branch a detached grade actually takes originally called `all_criteria_passed` inline, so the seeded field was written and never read. `tests/test_seed_from_prior_result.py` partitions every `EvaluationResult` field as CARRIED or RECOMPUTED and fails closed on a new one, and asserts the two `_select_gate()` call sites. A prior status that `FinalStatus.is_execution_fact` (TIMEOUT / ERROR / BUILD_FAILED / the budget stops) is **preserved**, never overwritten: grading may only move `NOT_GRADED` to SUCCESS/FAILURE, since it neither repeated nor observed the agent phase. Grader-host `environment_info` is preserved as flat `graded_by_*` scalars rather than overwriting the run's (flat, not a nested sub-dict: `environment_info` is rendered as a flat map by the HTML report and typed as one by the evalboard, so a nested capture prints as a Python dict repr). The route recorder follows the same rule: on a detached grade it writes `graded_by_api_routing` / `graded_by_eval_routing` and leaves the run's `api_routing` alone — writing in place contradicted the "prior wins" contract and left a self-contradictory record (a direct route named beside the run's stale `aws_region`/`bedrock_model`). Two other parity fixes: `command_base_path` is now persisted into `environment_info` by `_sync_sandbox_command_path_with_agent` and restored in the evaluate-only branch (closing the PATH gap that method's docstring already named), and `_join_litellm_actual_cost` **skips** when `prior_result` is set (its join keys on a per-Orchestrator nonce the prior turns never carried, so it would clobber already-correct costs). The verdict is written back into the run's `task.json`, with the pre-grade record kept as `task.execute.json` — that in-place write is what makes plain `coder-eval aggregate ` rebuild a graded `run.json` with **zero** new code. **`Sandbox.adopt(workspace)`** is the grade-in-place primitive: it reuses `setup`'s adoption half but skips every *materializing* step (`_setup_template`, `_generate_cli_recorders`, venv/package installs, the destructive `$HOME` remediation), running only non-mutating derivation (mock-dir `+x`, venv *discovery*, plugin-tools pin); `_cleanup_on_exit` stays False so an adopted tree is never moved or deleted, and `Sandbox.was_adopted` is set — the Orchestrator reads it to SKIP the `pre_run`/`post_run` hooks (`run()` calls them unconditionally with `cwd = sandbox_dir`, and several in-tree tasks stage fixtures there with `cp -a /app/[!.]* "$PWD/"`, which would overwrite the agent's deliverables before the criteria read them) and to KEEP `sandbox_path` in the `PreservationMode.NONE` cleanup arm (an adopted tree survives cleanup, so the path is not stale). The hooks' recorded results are carried from the prior run instead. In-place is **more correct**, not merely faster: `_setup_template` filters the copy through `_should_ignore_template_file`, which drops `node_modules` / `dist` / `build` / `.venv` / `.git`, so on the copy path a criterion like `test -f dist/bundle.js` fails as a *copying artifact* rather than as a verdict (verified: 0.00 "does not exist" on copy vs 1.00 in place). Defaults: in-place for a run dir, copy for a bare work dir (criteria can mutate it and it is the user's own tree); `--in-place`/`--copy` override. `adopt` hard-errors on `driver: docker` (a container workspace is unreachable from the host), and grading a `driver: docker` task is REFUSED outright unless `--allow-host-grading` is passed — the earlier behavior silently rewrote the driver to `tempdir`, which ran a container task's criteria against a host filesystem lacking `/verifier` and the image's toolchain (FAILURE for a trajectory `run` scored 1.0, plus `rm -rf /verifier` unsandboxed on the grading machine) and neutralized `adopt`'s own docker guard; an opted-in row is stamped `graded_on_host` so it is never silently comparable with a container-graded one (lint rule CE051). A re-grade refuses on a `reference_digest` mismatch — the digest is persisted into `environment_info` at staging time by `_stage_reference` (it shipped once as a read with no writer anywhere, so the guard was dead code), and `verify_reference_unchanged` now takes the task file it resolves against and RAISES on a vanished or unresolvable reference instead of returning silently. That comparison digests a STAGED copy of the source, not the raw tree: the recorded digest is taken over the staged copy, which `stage_reference_dir` filters through `REFERENCE_COPY_IGNORE` (`.git`), so digesting the source directly compared two differently-filtered trees and reported a permanent false mismatch for any reference that is a git checkout — exactly the case the ignore list exists for. **The recorded config is untrusted input**: `evaluate ` rebuilds the task from a shareable artifact, so a rebuilt config that carries shell (`run_command` criteria, `agent_judge`, `uipath_eval`, and — only on the `--copy` path, since the in-place path skips them — `pre_run`/`post_run`) is REFUSED unless `--allow-recorded-commands` is passed. A warning is not a control: it prints as the command is already being prepared. Passing the task file explicitly (`evaluate `) also bypasses it, since that config came from the operator. The workspace fallback `artifacts / prior.task_id` is containment-checked like its `sandbox_path` sibling (`task_id` is an unvalidated string, and `"../../.."` joins to a real directory `is_dir()` confirms), `_sanitize_restored_path` drops relative entries (they resolve against the grader's cwd) and anything inside the run dir rather than only the workspace, and `write_text_atomic` opens its temp file `O_EXCL|O_NOFOLLOW` — a pre-planted `task.json.tmp` symlink otherwise bypassed the write-back's destination symlink guard entirely. NOTE the Typer command is a thin wrapper over `run_evaluation(...)`, which has real Python defaults — calling a Typer command function in-process hands unspecified options an `OptionInfo` sentinel, which silently made `in_place=None` truthy. - **`--resume` is command-relative**: `partition_for_resume(tasks, *, grade)` returns a four-way `ResumePartition` (`to_run` / `to_grade` / `prior_results` / `prior_resolved`), because **"finished" is not absolute — it depends on what the resuming command still owes the task**. A `NOT_GRADED` row carries a final status, so the original "has any final status" test called it complete: right for `execute --resume` (it finished executing), and wrong for `run --resume`, which was asked to grade and would instead report "already complete", grade nothing, and **exit 0**. Under `grade=True` those rows route to `to_grade`, where `_grade_resumed_tasks` runs the criteria against the trajectory and workspace already on disk via `orchestration/regrade.py::regrade_in_place` — reusing the agent spend, which is the entire reason `execute` and `run` are separate. The carve-out is **only** for `NOT_GRADED`: `FAILURE`/`ERROR` stay complete under both commands (resume has never retried failures — delete the task.json), and `clear_rerun_artifacts` deliberately skips `to_grade`, whose artifacts are the very thing being graded. A per-task grading failure is warned, STAMPED onto the folded-back row's `error_message` (the console line alone is not durable), and folded back in with its ORIGINAL ungraded result, so one bad row neither aborts the resume nor vanishes from run.json — and the exit gate counts `tasks_not_graded` **when `grade` is True**, so a `run` that graded nothing exits non-zero instead of telling CI the suite is fine. Under `execute` an ungraded row is the expected outcome and never fails the command. `grade` is in `_FINGERPRINT_DIFF_EXEMPT` because `execute` → `run --resume` is a supported flow, not config drift — and the warning's "already-finalized tasks keep their original-config results" text is actively wrong for it. **`orchestration/regrade.py` is the single implementation** shared by that path and `evaluate`'s run-dir mode, which DELEGATES to `regrade_in_place` rather than restating it (it originally hand-built its own Sandbox + Orchestrator and had already drifted — hardcoding `replicate_index=0`, so every replicate but the first was relabelled — which is exactly how two copies become two verdicts for the same run); it raises plain `RegradeError`, which the CLI wraps, since `orchestration/` must not import the CLI layer (CE004). One fidelity rule it enforces: a re-graded row keeps the **agent run's** `started_at`/`duration_seconds`, not the grading pass's — a 10-minute run re-graded in 2s would otherwise report 2s into `average_duration`, the report tables and the evalboard; the grading cost is preserved separately as `environment_info["grading_duration_seconds"]`. - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `stop_early.decide_within`, deliberately lives on `LiveSuccessCriterion` instead (see below) — the watcher must attribute a decision-step timeout to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express. - **Early stop on criterion (opt-in, per-criterion arming)**: a `stop_early:` block (`StopEarlyPolicy`) on a criterion ends a single-shot run early once the run's **armed** criteria decide the outcome, so a raised `max_turns` isn't wasted on the smoke flavor. The block's PRESENCE is the arming and alone activates the watcher — there is **no run-level master switch**: `run_limits.stop_early: false` is the run-level KILL SWITCH that force-disarms every block (the one-line experiment-variant/`-D` override for an authoritative full run), and `run_limits.stop_early: true` (the removed master arm) is a hard `EarlyStopConfigError` at resolution. The block exists on `LiveSuccessCriterion` only (currently `skill_triggered`, `command_executed` — so arming an unobservable criterion is unrepresentable, a pydantic extra-forbid error). Arming carries one implicit trigger (a native live-fail may fail-stop the run); its keys refine it: `on_pass: stop` (pass-stop the moment the criterion live-passes; default `continue` just latches) and `decide_within: N` (still undecided after N tool-call steps latches an **effective fail**, fed through the same fail-stop rule, reported as `decision_budget_exceeded` — an ordinary weighted fail, NOT a gate-bypassing force-fail; cumulative across retry attempts of the same turn). A trigger whose polarity the instance can't decide (per the abstract, checker-independent `live_decidable_polarities()`, a pure function of the criterion's own fields, paired with the checker's `live_verdict` override by lint rule CE025, a registry-based whole-tree check) is **inert by design** — one dataset-fanned YAML line serves both positive rows (pass/timeout live) and distractor rows (fail live). Verdicts **latch**: once a criterion decides, its `live_verdict` is never polled again. Stop rule is weighted, not strict-boolean: `run_limits.stop_early_gate_threshold` (default `1.0`, reproducing strict-AND behavior exactly) is the minimum weighted score (`Σ weight·score / Σ weight` over the armed subset) required to pass; a fail-stop fires once the armed set's **ceiling** (best case for everything still undecided) can no longer reach the threshold — so a low-weight fail or timeout that can't doom the gate is absorbed and the run continues — and is **deferred while any pass-capable armed criterion is undecided** (a distractor misfire never truncates a positive row's recall signal); a pass-stop fires once the `on_pass: stop` subset's **floor** (worst case) already meets the threshold, and is symmetrically **deferred while any pass-capable armed criterion outside the `on_pass: stop` subset is undecided** (so an early pass never freezes a sibling `on_pass: continue` criterion's signal out of the trajectory). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a kill-switched (`stop_early: false`) run. Driven by `orchestration/early_stop.py::EarlyStopWatcher` (built when `early_stop_active(task)`: ≥1 armed criterion, kill switch not thrown) through the agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. Gating is **FIRED-ONLY**: a run the watcher actually cut gates on the **armed subset** via the weighted `EvaluationResult.armed_criteria_passed`; a run that completes naturally — armed or not — gates strict-AND via `all_criteria_passed`, so adding a block never changes the verdict of a run it didn't cut. Note the gate keys on the watcher having FIRED (`result.early_stop is not None`), not on confirmed truncation — an agent that ignores `should_stop`, or a stop firing on the final message, still gates armed-only. Every resolution-time guardrail violation is a hard error at resolution (plan *and* run); the one load-time case — a `stop_early:` block on a non-live criterion — is a pydantic schema error at task load, which the run surface reports as a skipped task like any other malformed task. A runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo` (incl. `gate_threshold` at stop time), report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. No blocks anywhere ⇒ behavior byte-for-byte unchanged. @@ -223,7 +223,7 @@ make plugin-reference # the plugin's bundled criteria reference from the models Editing `src/coder_eval/pricing.py` means editing `evalboard/lib/pricing.ts` too — it is a hand-copied mirror, and `evalboard/lib/__tests__/pricing-parity.test.ts` fails the build on drift in either direction. -Recent additions, each traceable to a shipped defect: **CE047** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's). +Recent additions, each traceable to a shipped defect: **CE047** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE049** (never coalesce a possibly-unmeasured score to a numeric literal — `score or 0.0` publishes "measured and scored zero" while meaning "never measured", which is how an ungraded night reached four unfiltered `avg(Score)` dashboards as a real zero), **CE050** (no untyped `getattr` probe for a discriminated-union field — pyright cannot see the string, so a rename degrades the guard to a permanent no-op; scoped to criterion-shaped receivers because `command`/`tool`/`prompt` are far too common to flag on their own), **CE051** (a sandbox driver may not be rewritten silently — the driver IS the isolation boundary, so a downgrade must be an explicit, stamped, operator-visible decision), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's). When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033–CE036 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE036 enforces the `live_verdict` determinism + monotonicity contract (`criteria/base.py`) that `EarlyStopWatcher`'s latching, deferred fail-stop, and flip-attribution silently depend on: monotonicity over arbitrary Python is undecidable, so instead of a static check it REPLAYS each live criterion against every prefix of recorded trajectories (`tests/lint/live_verdict_contract.py::CASES`) — on the authored ordering AND under seeded shuffles (`permuted_violations`, which catch order-sensitive bugs the authored walk misses) — and asserts the property directly, plus registry-derived coverage — every `LiveSuccessCriterion` in the union must have cases, and every polarity its instances claim via `live_decidable_polarities()` must actually be reached by one (otherwise a single always-`undecided` fixture would "cover" a type while proving nothing). Adding a live criterion therefore means adding `ContractCase`s in the same change. CE035 resolves every `steps..outputs.` / `needs..outputs.` reference in `.github/workflows/**` to a writer that actually produces that key — GitHub expands an unwritten output to the empty string, so a typo degrades a gate silently and actionlint models `steps.*.outputs` as an open string map. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.) diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index c9405ae6..35bc96d7 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -235,8 +235,9 @@ the same weighted armed gate as a native fail), ## `variant.json` — `VariantAggregate` A single aggregate (not wrapped): `variant_id`, `tasks_run`, `tasks_succeeded`, -`tasks_failed`, `tasks_error`, `tasks_not_graded` (same sum-to-`tasks_run` invariant), `average_score` -(the mean over **graded** rows only), +`tasks_failed`, `tasks_error`, `tasks_not_graded` (same sum-to-`tasks_run` invariant), +`average_score` (`float | None` — the mean over **measured** rows: an errored row counts +as `0.0`, an ungraded one leaves both sides; `null` when nothing was measured), `average_duration`, `total_tokens`, `replicate_count`, `tasks_token_budget_exceeded`, `tasks_cost_budget_exceeded`. @@ -247,7 +248,8 @@ The cross-variant summary: - `experiment_id`, `description`, `variant_ids`. - `task_summaries: list[TaskExperimentSummary]` — each `{task_id, variant_results, best_variant, is_tie, score_spread, replicate_count}`, - where each `VariantResult` carries `{variant_id, task_id, weighted_score, + where each `VariantResult` carries `{variant_id, task_id, weighted_score` (`float | + None` — `null` on an ungraded or errored row)`, final_status, duration_seconds, total_tokens, iteration_count, total_assistant_turns, reference_similarity, replicate_index, replicate_count}`. - `variant_aggregates: dict[str, VariantAggregate]` — keyed by variant id. @@ -267,7 +269,8 @@ Written for dataset-backed suites; its `passed` flag drives the CI exit code. | --- | --- | --- | | `suite_id` / `variant_id` | `str` | Identity. | | `rows_total` / `rows_passed` / `rows_failed` / `rows_error` / `rows_not_graded` | `int` | Row counts. **Invariant:** the four category counts sum to `rows_total`. `rows_not_graded` defaults to `0`. | -| `pass_rate` | `float` | `rows_passed / (rows_total - rows_not_graded)` — ungraded rows leave both sides, matching `RunSummary.pass_rate`. | +| `pass_rate` | `float \| null` | `rows_passed / rows_graded` — ungraded rows leave both sides, matching `RunSummary.pass_rate`. `null` when nothing was graded (0/0 is unknown, not 0%). | +| `rows_graded` | `int` | `rows_total - rows_not_graded`. The denominator above, serialized so a consumer never has to re-derive it. | | `average_weighted_score` | `float \| null` | Mean row score. | | `criterion_stats` | `list[{criterion_type, rows_evaluated, average_score, error_count}]` | Per-criterion summary. | | `failed_samples` | `list[FailedRowSummary]` | Capped at 20 (`{row_id, task_id, final_status, weighted_score, failure_reasons, error_message, task_json_relpath, replicate_index}`). | diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index bbd65286..2a72520d 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -172,7 +172,24 @@ judges with trajectory) score exactly as they would have during the run. It writes the verdict back into the run's `task.json` and keeps the pre-grade record beside it as `task.execute.json`. Writing back in place is what makes -`aggregate` free — no new flag, no second copy of the results. +`aggregate` free — no new flag, no second copy of the results. If grading itself +crashes, the ungraded record is put back: `ERROR` counts as complete for both +commands, so an errored row could never be graded again. + +**A run directory is untrusted input.** It is a shareable artifact — the whole +point of the detached flow is that one machine executes and another grades — and +rebuilding the task from it means the run dir decides what runs on your host, +with your environment. So two things are refused rather than assumed: + +- A recorded config that carries shell (`run_command` criteria, `agent_judge`, + `uipath_eval`, and on the `--copy` path `pre_run`/`post_run`) needs + `--allow-recorded-commands`. The message names every command first. Passing the + task file explicitly bypasses this — that config came from you. +- A run made with `driver: docker` needs `--allow-host-grading`. Grading cannot + start a container, and such a task's criteria address container paths and + toolchains; on your host they score `0.0` for a run that passed. An opted-in + row is stamped `graded_on_host` in `environment_info` so it is never silently + compared with a container-graded one. Passing a task file **over** a run directory re-grades it with different criteria, reusing the trajectory and workspace of a run you already paid for: diff --git a/evalboard/app/_overview/window-summary.tsx b/evalboard/app/_overview/window-summary.tsx index 29c06245..e67ace75 100644 --- a/evalboard/app/_overview/window-summary.tsx +++ b/evalboard/app/_overview/window-summary.tsx @@ -54,8 +54,8 @@ export function WindowSummary({ isFiltered: boolean; }) { const pct = - totals.tasksRun > 0 - ? (totals.tasksSucceeded / totals.tasksRun) * 100 + totals.tasksGraded > 0 + ? (totals.tasksSucceeded / totals.tasksGraded) * 100 : null; const scope = isFiltered ? `matching · last ${window}` diff --git a/evalboard/app/page.tsx b/evalboard/app/page.tsx index 8537c1ac..7a66f245 100644 --- a/evalboard/app/page.tsx +++ b/evalboard/app/page.tsx @@ -379,8 +379,12 @@ export default async function Page({ {listing.rows.map((r) => { const total = r.tasksRun; - const pct = total - ? (r.tasksSucceeded / total) * 100 + // Ungraded rows (`coder-eval execute`) leave BOTH + // sides: the rate divides by tasksGraded, while the + // Tasks column still reports everything that ran. + const graded = r.tasksGraded; + const pct = graded + ? (r.tasksSucceeded / graded) * 100 : null; return ( - {r.tasksSucceeded}/{total} + {r.tasksSucceeded}/{graded} @@ -517,8 +521,9 @@ export default async function Page({ {adhoc.rows.map((r) => { const total = r.tasksRun; - const pct = total - ? (r.tasksSucceeded / total) * 100 + const graded = r.tasksGraded; + const pct = graded + ? (r.tasksSucceeded / graded) * 100 : null; return ( - {r.tasksSucceeded}/{total} + {r.tasksSucceeded}/{graded} diff --git a/evalboard/app/path-to-ga/__tests__/task-table.test.tsx b/evalboard/app/path-to-ga/__tests__/task-table.test.tsx index c67284a6..179100a2 100644 --- a/evalboard/app/path-to-ga/__tests__/task-table.test.tsx +++ b/evalboard/app/path-to-ga/__tests__/task-table.test.tsx @@ -12,6 +12,7 @@ function row(overrides: Partial = {}): TagTaskRow { skill: "uipath-maestro-flow", appearances: 20, matureSkips: 0, + ungraded: 0, executed: 20, passRate: 90, latestStatus: "SUCCESS", diff --git a/evalboard/app/runs/[id]/__tests__/task-grid.test.tsx b/evalboard/app/runs/[id]/__tests__/task-grid.test.tsx index c840715f..27953ed3 100644 --- a/evalboard/app/runs/[id]/__tests__/task-grid.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/task-grid.test.tsx @@ -408,7 +408,7 @@ describe("TaskGrid — replicates", () => { expect(reptaskLinks[0]).toHaveAttribute("href", "/runs/r1/reptask?r=0"); // k/N ✓ badge: all 3 replicates passed → "3 of 3", GREEN. solo (single // run) shows no badge. - const badge = within(table).getByTitle(/3 of 3 replicates passed/i); + const badge = within(table).getByTitle(/3 of 3 graded replicates passed/i); expect(badge.textContent?.replace(/\s/g, "")).toBe("3/3✓"); expect(badge.className).toContain("text-green-700"); }); @@ -426,7 +426,7 @@ describe("TaskGrid — replicates", () => { ); const table = screen.getByRole("table"); // 1 of 3 replicates passed → shows the pass count, AMBER (partial). - const badge = within(table).getByTitle(/1 of 3 replicates passed/i); + const badge = within(table).getByTitle(/1 of 3 graded replicates passed/i); expect(badge.textContent?.replace(/\s/g, "")).toBe("1/3✓"); expect(badge.className).toContain("text-amber-700"); }); @@ -442,7 +442,7 @@ describe("TaskGrid — replicates", () => { />, ); const table = screen.getByRole("table"); - const badge = within(table).getByTitle(/0 of 2 replicates passed/i); + const badge = within(table).getByTitle(/0 of 2 graded replicates passed/i); expect(badge.textContent?.replace(/\s/g, "")).toBe("0/2✓"); expect(badge.className).toContain("text-red-700"); }); @@ -469,7 +469,7 @@ describe("TaskGrid — replicates", () => { expect(link).toHaveAttribute("href", "/runs/r1/t?r=1"); // Badge still shows the true ratio. expect( - within(tr).getByTitle(/1 of 2 replicates passed/i).textContent?.replace(/\s/g, ""), + within(tr).getByTitle(/1 of 2 graded replicates passed/i).textContent?.replace(/\s/g, ""), ).toBe("1/2✓"); }); }); diff --git a/evalboard/app/runs/[id]/__tests__/ungraded-golden.test.ts b/evalboard/app/runs/[id]/__tests__/ungraded-golden.test.ts new file mode 100644 index 00000000..3cd06210 --- /dev/null +++ b/evalboard/app/runs/[id]/__tests__/ungraded-golden.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, test } from "vitest"; +import type { TaskResultSummary } from "@/lib/runs"; +import { perTaskGradedCounts, perTaskPassCounts } from "@/lib/status"; +import { taskVariantKey } from "@/lib/variants"; +import { computeRunMetrics, computeVariantMetrics } from "../run-view"; + +// One invariant, asserted through every rate surface a `coder-eval execute` run +// touches: an UNGRADED row leaves BOTH sides of the rate. +// +// This is arithmetic over a data shape, not a code pattern, so no lint rule +// catches it — and the first ungraded sweep proved a correct guard can still be +// applied to the wrong denominator: `pct` divided by the graded count while the +// LABEL beside it read `passed / total`, so 8 graded passes next to 2 ungraded +// rows rendered "80%" beside "8 / 12", and a plain 12-task execute run rendered +// a red "0% 0 / 12". + +function row( + taskId: string, + extra: Partial = {}, +): TaskResultSummary { + return { + taskId, + variantId: null, + replicateIndex: null, + status: "SUCCESS", + weightedScore: 1.0, + durationSeconds: 1.0, + totalCostUsd: 0.1, + actualCommands: null, + totalTurns: null, + expectedTurns: null, + expectedSeconds: null, + hasFinalReply: false, + inputTokens: null, + outputTokens: null, + cacheCreationTokens: null, + cacheReadTokens: null, + model: null, + tags: [], + skill: null, + matureSkipped: false, + ...extra, + }; +} + +const ungraded = (taskId: string, extra: Partial = {}) => + row(taskId, { status: "NOT_GRADED", weightedScore: null, ...extra }); + +describe("a fully ungraded run reports 0 of 0, never 0%", () => { + const tasks = Array.from({ length: 12 }, (_, i) => ungraded(`t${i}`)); + const m = computeRunMetrics(tasks); + + test("every row is bucketed as ungraded, none as failed", () => { + expect(m.total).toBe(12); + expect(m.ungraded).toBe(12); + expect(m.failed).toBe(0); + expect(m.errored).toBe(0); + expect(m.passed).toBe(0); + }); + + test("the denominator the tile renders is zero, so the tile reads neutral", () => { + // `graded`, not `total`. The tile's tone is null when this is 0, which is + // what stops a clean execute run rendering as a red measured 0%. + expect(m.graded).toBe(0); + }); + + test("the per-task rollup does not score them as failures", () => { + // 0 of 0 tasks, not 0 of 12: an all-ungraded task is absent from the + // rollup entirely rather than appearing as a task that failed. + expect(m.taskTotal).toBe(0); + expect(m.taskPassed).toBe(0); + expect(m.taskFailed).toBe(0); + }); +}); + +describe("a mixed run divides by the graded rows on both halves of the tile", () => { + const tasks = [ + ...Array.from({ length: 8 }, (_, i) => row(`p${i}`)), + row("f0", { status: "FAILURE", weightedScore: 0 }), + row("f1", { status: "FAILURE", weightedScore: 0 }), + ungraded("u0"), + ungraded("u1"), + ]; + const m = computeRunMetrics(tasks); + + test("pct and its label describe the same sample", () => { + expect(m.total).toBe(12); + expect(m.graded).toBe(10); + expect(m.passed).toBe(8); + expect(m.pct).toBeCloseTo(80); + // The pair the tile renders: "80% 8 / 10". Dividing by `total` here is + // the bug this asserts against — it produced "80% 8 / 12". + expect(m.passed / m.graded).toBeCloseTo(m.pct / 100); + }); + + test("ungraded rows are not counted as failures", () => { + expect(m.failed).toBe(2); + expect(m.failedTotal).toBe(2); + }); +}); + +describe("repeats: an ungraded replicate leaves both sides of the k/N badge", () => { + const tasks = [ + row("t", { replicateIndex: 0 }), + ungraded("t", { replicateIndex: 1 }), + ungraded("u", { replicateIndex: 0 }), + ungraded("u", { replicateIndex: 1 }), + ]; + + test("the badge reads 1/1, not 1/2", () => { + const passes = perTaskPassCounts(tasks); + const graded = perTaskGradedCounts(tasks); + const key = taskVariantKey({ taskId: "t", variantId: null }); + expect(passes.get(key)).toBe(1); + expect(graded.get(key)).toBe(1); + }); + + test("a task whose every replicate was ungraded is absent, not 0/2", () => { + const graded = perTaskGradedCounts(tasks); + expect(graded.has(taskVariantKey({ taskId: "u", variantId: null }))).toBe( + false, + ); + }); +}); + +describe("variants: each arm's rate excludes its own ungraded rows", () => { + const tasks = [ + row("t0", { variantId: "a" }), + ungraded("t1", { variantId: "a" }), + row("t0", { variantId: "b", status: "FAILURE", weightedScore: 0 }), + ungraded("t1", { variantId: "b" }), + ]; + const rows = computeVariantMetrics(tasks); + + test("arm a is 100% of one graded task, not 50% of two", () => { + const a = rows.find((r) => r.variantId === "a")!.metrics; + expect(a.graded).toBe(1); + expect(a.taskTotal).toBe(1); + expect(a.taskPassed).toBe(1); + }); + + test("arm b is a measured 0% — an ungraded row must not soften a real failure", () => { + const b = rows.find((r) => r.variantId === "b")!.metrics; + expect(b.graded).toBe(1); + expect(b.taskTotal).toBe(1); + expect(b.taskPassed).toBe(0); + }); +}); diff --git a/evalboard/app/runs/[id]/run-view.tsx b/evalboard/app/runs/[id]/run-view.tsx index 0e48d756..68398baf 100644 --- a/evalboard/app/runs/[id]/run-view.tsx +++ b/evalboard/app/runs/[id]/run-view.tsx @@ -6,7 +6,7 @@ import type { ActivationScore, TaskResultSummary } from "@/lib/runs"; import type { ReviewIndexEntry } from "@/lib/reviews-types"; import { fmtDuration, humanizeTaskId } from "@/lib/format"; import { passBarClass, passClass } from "@/lib/pass-rate"; -import { perTaskPassCounts, statusCategory } from "@/lib/status"; +import { assertNever, perTaskPassCounts, statusCategory } from "@/lib/status"; import { DEFAULT_VARIANT_ID, taskVariantKey, @@ -41,6 +41,11 @@ export interface RunMetrics { // Rows that ran but were never scored (`coder-eval execute`). Excluded from // both sides of `pct`, so a fully ungraded run reports 0 of 0, not 0%. ungraded: number; + // `total - ungraded`: the pass-rate denominator. Carried on the metrics + // rather than re-derived at each tile — the tile used to divide by `graded` + // while LABELLING the result `passed / total`, so 8 of 10 graded beside 2 + // ungraded rendered "80% 8 / 12". + graded: number; failedTotal: number; pct: number; // Per-task view of pass rate for repeated runs: distinct task_ids, and how @@ -81,15 +86,30 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics { const costSamples: number[] = []; const durSamples: number[] = []; for (const t of tasks) { + // A `switch` with an assertNever default, not an if/else chain with a + // catch-all `else failed++`. That chain is what made adding "ungraded" + // a SILENT change: anything not a pass or an error was counted as a + // failure and kept in the denominator, and no compiler saw it. With + // this shape a fifth StatusCategory fails `tsc --noEmit` right here. const cat = statusCategory(t.status); - if (cat === "passed") passed++; - else if (cat === "error") errored++; - // An ungraded row (`coder-eval execute`) was never scored. It leaves - // BOTH sides of the rate — the `else failed++` below would otherwise - // count it as a failure AND keep it in the denominator, rendering a - // clean execute run as 0% pass, N failed. - else if (cat === "ungraded") ungraded++; - else failed++; + switch (cat) { + case "passed": + passed++; + break; + case "error": + errored++; + break; + case "ungraded": + // Never scored, so it leaves BOTH sides of the rate. + ungraded++; + break; + case "failed": + case "unknown": + failed++; + break; + default: + assertNever(cat); + } if (t.matureSkipped) continue; if (t.totalCostUsd != null) { cost += t.totalCostUsd; @@ -107,11 +127,15 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics { failed, errored, ungraded, + graded, failedTotal: failed + errored, pct: graded ? (passed / graded) * 100 : 0, ...(() => { // Per-task rollup (any replicate passed → task passed) via the shared // helper, so the run tile and the grid badge apply the same rule. + // perTaskPassCounts already drops ungraded replicates, so a task + // whose replicates were all ungraded is absent from the map — which + // is what keeps it out of BOTH taskTotal and taskPassed. const perTask = perTaskPassCounts(tasks); const taskPassed = [...perTask.values()].filter((n) => n > 0).length; return { @@ -555,11 +579,15 @@ export function RunView({ const passed = hasRepeats ? metrics.taskPassed : metrics.passed; + // metrics.graded, NOT metrics.total: `pct` divides by + // the graded count, so labelling it with every row that + // ran renders a mismatched pair (and a red 0% next to + // "0 / 12" for a clean `coder-eval execute` run). const totalN = hasRepeats ? metrics.taskTotal - : metrics.total; - // null when nothing ran, so an empty run reads neutral - // rather than as a measured 0%. + : metrics.graded; + // null when nothing was MEASURED, so an empty or fully + // ungraded run reads neutral rather than as a 0%. const tone = totalN > 0 ? pct : null; return ( <> diff --git a/evalboard/app/runs/[id]/task-grid.tsx b/evalboard/app/runs/[id]/task-grid.tsx index 4209feb2..6d3fb7b9 100644 --- a/evalboard/app/runs/[id]/task-grid.tsx +++ b/evalboard/app/runs/[id]/task-grid.tsx @@ -14,7 +14,7 @@ import { MaturePill, StatusPill, } from "@/lib/pills"; -import { isPassStatus, perTaskPassCounts, statusSortRank } from "@/lib/status"; +import { isPassStatus, perTaskGradedCounts, perTaskPassCounts, statusSortRank } from "@/lib/status"; import { DEFAULT_VARIANT_ID, taskVariantKey, variantsOf } from "@/lib/variants"; import { displayedTurns, @@ -182,6 +182,7 @@ function TaskIdCell({ className, matureSourceRuns, replicateCount = 1, + replicateGradedCount = 0, replicatePassCount = 0, sourceId, }: { @@ -194,6 +195,8 @@ function TaskIdCell({ // >1 the row collapses to a single entry with a k/N ✓ badge; the per-run // detail is reachable from the task page's run selector (?r=NN). replicateCount?: number; + // How many of those replicates were graded — the badge's denominator. + replicateGradedCount?: number; // How many of those replicates passed — shown as k/N and color-coded. replicatePassCount?: number; }) { @@ -238,12 +241,12 @@ function TaskIdCell({ return ( {humanizeTaskId(t.taskId)} - {replicateCount > 1 && ( + {replicateCount > 1 && replicateGradedCount > 0 && ( - {replicatePassCount}/{replicateCount} ✓ + {replicatePassCount}/{replicateGradedCount} ✓ )} @@ -565,6 +568,13 @@ export function TaskGrid({ // page's pass-rate tile all apply the same "any replicate passed" rule. const replicatePassCounts = useMemo(() => perTaskPassCounts(tasks), [tasks]); + // The badge's DENOMINATOR. Not `replicateCounts`, which counts every + // replicate: an ungraded replicate (`coder-eval execute`) was never scored, + // so counting it there rendered a red "0/2 ✓" with the tooltip "0 of 2 + // replicates passed" for a run nothing was wrong with. Zero (every + // replicate ungraded) suppresses the badge entirely. + const replicateGradedCounts = useMemo(() => perTaskGradedCounts(tasks), [tasks]); + // Collapse replicates to one row per (variant, task): repeated runs share a // taskId, so the grid shows a single entry with a k/N ✓ badge; the per-run // detail is selectable on the task page. The representative is chosen so its @@ -752,6 +762,9 @@ export function TaskGrid({ replicateCount={ replicateCounts.get(taskVariantKey(t)) ?? 1 } + replicateGradedCount={ + replicateGradedCounts.get(taskVariantKey(t)) ?? 0 + } replicatePassCount={ replicatePassCounts.get(taskVariantKey(t)) ?? 0 } @@ -913,6 +926,9 @@ export function TaskGrid({ replicateCount={ replicateCounts.get(taskVariantKey(t)) ?? 1 } + replicateGradedCount={ + replicateGradedCounts.get(taskVariantKey(t)) ?? 0 + } replicatePassCount={ replicatePassCounts.get(taskVariantKey(t)) ?? 0 } diff --git a/evalboard/app/scribe/run-table.tsx b/evalboard/app/scribe/run-table.tsx index 22fc0984..d3e767eb 100644 --- a/evalboard/app/scribe/run-table.tsx +++ b/evalboard/app/scribe/run-table.tsx @@ -11,8 +11,10 @@ import { TableScroll } from "../_components/scroll-table"; type ScribeRow = RunListingRow & { title?: string | null }; function passPct(row: ScribeRow): number | null { - if (row.tasksRun === 0) return null; - return (row.tasksSucceeded / row.tasksRun) * 100; + // tasksGraded, not tasksRun: an ungraded row (`coder-eval execute`) was + // never scored, so it leaves both sides of the rate. + if (row.tasksGraded === 0) return null; + return (row.tasksSucceeded / row.tasksGraded) * 100; } export function ScribeRunTable({ @@ -120,7 +122,7 @@ export function ScribeRunTable({ : "—"} - {row.tasksSucceeded}/{row.tasksRun} + {row.tasksSucceeded}/{row.tasksGraded} {fmtDuration(row.taskDurationSeconds)} diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts index d012d39d..3f4f4f7b 100644 --- a/evalboard/lib/__tests__/overview.test.ts +++ b/evalboard/lib/__tests__/overview.test.ts @@ -44,6 +44,7 @@ describe("summarizeListing", () => { id: "r", tasksSucceeded: 0, tasksRun: 0, + tasksGraded: 0, totalCostUsd: null, taskDurationSeconds: null, ...overrides, @@ -57,6 +58,7 @@ describe("summarizeListing", () => { costPartial: false, tasksSucceeded: 0, tasksRun: 0, + tasksGraded: 0, durationSeconds: null, durationPartial: false, }); diff --git a/evalboard/lib/__tests__/status-parity.test.ts b/evalboard/lib/__tests__/status-parity.test.ts new file mode 100644 index 00000000..0ade2830 --- /dev/null +++ b/evalboard/lib/__tests__/status-parity.test.ts @@ -0,0 +1,77 @@ +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, test } from "vitest"; +import { statusCategory, type StatusCategory } from "../status"; + +// Drift guard: lib/status.ts is a hand-copied mirror of coder_eval's +// `FinalStatus.category` (src/coder_eval/models/enums.py). Python's side is +// guarded there by `assert set(_STATUS_CATEGORIES) == set(FinalStatus)`; this +// mirror was not, which is how NOT_GRADED came to be categorized as "unknown" +// and then counted as a failure by every rate helper in the app. +// +// status.test.ts covers the mapping, but it iterates a HAND-MAINTAINED record — +// so it can never fail when Python adds a tenth member. This file parses the +// Python table instead, following the lib/__tests__/pricing-parity.test.ts +// precedent, so the next status added upstream breaks the build here rather +// than silently rendering as grey "unknown" and inflating a denominator. + +const here = dirname(fileURLToPath(import.meta.url)); +const PY_PATH = resolve(here, "../../../src/coder_eval/models/enums.py"); + +// Match: `FinalStatus.SUCCESS: "succeeded",` inside _STATUS_CATEGORIES. +const ROW_RE = /FinalStatus\.([A-Z_]+):\s*"(succeeded|failed|error|ungraded)"/g; + +// Python's four buckets -> the TS union. The names differ on ONE member +// ("succeeded" vs "passed"), deliberately: "passed" is the word the UI uses. +// Spelled out here so a rename on either side is a failure, not a silent +// fall-through to "unknown". +const PY_TO_TS: Record = { + succeeded: "passed", + failed: "failed", + error: "error", + ungraded: "ungraded", +}; + +function parsePythonCategories(): Record { + const src = readFileSync(PY_PATH, "utf8"); + const start = src.indexOf("_STATUS_CATEGORIES"); + expect(start, "_STATUS_CATEGORIES not found in enums.py").toBeGreaterThan( + -1, + ); + // Bound the scan to that dict so _EXECUTION_FACT_STATUSES and the docstrings + // below it cannot contribute phantom rows. + const end = src.indexOf("\n}", start); + const table = src.slice(start, end === -1 ? undefined : end); + + const out: Record = {}; + for (const m of table.matchAll(ROW_RE)) { + out[m[1]] = PY_TO_TS[m[2]]; + } + return out; +} + +describe("status.ts mirrors coder_eval FinalStatus.category", () => { + const py = parsePythonCategories(); + + test("the Python table was actually parsed", () => { + // A regex that silently matches nothing would make every assertion + // below vacuous — the exact failure mode this file exists to prevent. + expect(Object.keys(py).length).toBeGreaterThanOrEqual(9); + expect(py.SUCCESS).toBe("passed"); + expect(py.NOT_GRADED).toBe("ungraded"); + }); + + test.each(Object.entries(parsePythonCategories()))( + "%s categorizes as %s on both sides", + (status, expected) => { + expect(statusCategory(status)).toBe(expected); + }, + ); + + test("no member falls through to unknown", () => { + for (const status of Object.keys(py)) { + expect(statusCategory(status)).not.toBe("unknown"); + } + }); +}); diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index b3343b62..e8e26f51 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -18,7 +18,7 @@ import { withinTurnBudget } from "./turns"; import { humanizeTaskId } from "./format"; import { mapWithConcurrency } from "./concurrency"; import { DEFAULT_HARNESS, normalizeHarness, orderHarnesses } from "./harness"; -import { isPassStatus } from "./status"; +import { isGraded, isPassStatus } from "./status"; import { taskCarriesRepoTag } from "./tags"; import type { Window } from "./reviews-types"; @@ -135,7 +135,7 @@ export function timePerPassedTaskForTasks( tasks: RunOverviewTask[], ): number | null { const executed = tasks.filter((t) => !t.matureSkipped); - const passed = executed.filter((t) => t.status === "SUCCESS").length; + const passed = executed.filter((t) => isPassStatus(t.status)).length; if (!passed) return null; const total = executed.reduce((a, t) => a + (t.durationSeconds ?? 0), 0); return total > 0 ? total / passed : null; @@ -171,6 +171,10 @@ export interface RunListingRow { // Unfiltered, they are whole-run totals. tasksSucceeded: number; tasksRun: number; + // Rows that were actually scored. An ungraded row (`coder-eval execute`) + // leaves BOTH sides of every rate, so this — not tasksRun — is the pass-rate + // denominator. Equals tasksRun on any graded run. + tasksGraded: number; totalCostUsd: number | null; taskDurationSeconds: number | null; // Run-level harness (coder-eval AgentKind) for the Harness column; null on @@ -189,6 +193,7 @@ export interface RunListingTotals { costPartial: boolean; // some matched runs had no cost (sum understates) tasksSucceeded: number; tasksRun: number; + tasksGraded: number; // the pass-rate denominator; see RunListingRow.tasksGraded durationSeconds: number | null; // null when no matched run recorded a duration durationPartial: boolean; } @@ -201,11 +206,13 @@ export function summarizeListing(rows: RunListingRow[]): RunListingTotals { let costRuns = 0; let tasksSucceeded = 0; let tasksRun = 0; + let tasksGraded = 0; let durationSeconds = 0; let durationRuns = 0; for (const r of rows) { tasksSucceeded += r.tasksSucceeded; tasksRun += r.tasksRun; + tasksGraded += r.tasksGraded; if (r.totalCostUsd != null) { costUsd += r.totalCostUsd; costRuns += 1; @@ -220,6 +227,7 @@ export function summarizeListing(rows: RunListingRow[]): RunListingTotals { costPartial: costRuns > 0 && costRuns < rows.length, tasksSucceeded, tasksRun, + tasksGraded, durationSeconds: durationRuns > 0 ? durationSeconds : null, durationPartial: durationRuns > 0 && durationRuns < rows.length, }; @@ -786,7 +794,11 @@ export async function getOverview( runId: id, timestamp: date.getTime(), harness: runHarness, - successRate: (row.tasksSucceeded / row.tasksRun) * 100, + // Ungraded rows leave both sides; a fully ungraded run contributes + // no point rather than a fabricated 0%. + successRate: row.tasksGraded + ? (row.tasksSucceeded / row.tasksGraded) * 100 + : null, turnBudgetRate: turnBudgetRateForTasks(scoped.tasks), withinExpectedTimeRate: withinExpectedTimeRateForTasks( scoped.tasks, @@ -830,11 +842,16 @@ export interface TagTaskRow { appearances: number; // Of `appearances`, how many were mature carry-forwards (not executed). matureSkips: number; - // `appearances - matureSkips`: the denominator behind passRate, carried on - // the row rather than re-derived by the renderer so the percentage and the - // caption that names its sample size can never describe different rules. + // Of `appearances`, how many ran but were never scored (`coder-eval + // execute`). Like a mature skip, such a row leaves BOTH sides of the rate. + ungraded: number; + // `appearances - matureSkips - ungraded`: the denominator behind passRate, + // carried on the row rather than re-derived by the renderer so the + // percentage and the caption that names its sample size can never describe + // different rules. executed: number; - // 0-100 over EXECUTED appearances only (appearances - matureSkips). + // 0-100 over MEASURED appearances only + // (appearances - matureSkips - ungraded). // null when nothing in the window actually ran, so the UI shows "—" // rather than a measured-looking 0% or 100%. passRate: number | null; @@ -893,6 +910,7 @@ export function buildTagTaskRows(perRun: PerRun[], tag: string): TagTaskRow[] { skill: string | null; appearances: number; matureSkips: number; + ungraded: number; executedPasses: number; latestRunId: string; latestStatus: string | null; @@ -939,6 +957,7 @@ export function buildTagTaskRows(perRun: PerRun[], tag: string): TagTaskRow[] { skill: t.skill, appearances: 0, matureSkips: 0, + ungraded: 0, executedPasses: 0, latestRunId: id, latestStatus: t.status, @@ -950,6 +969,11 @@ export function buildTagTaskRows(perRun: PerRun[], tag: string): TagTaskRow[] { entry.appearances += 1; if (t.matureSkipped) { entry.matureSkips += 1; + } else if (!isGraded(t.status)) { + // Never scored, so it is neither a pass nor a miss. Counted like + // a mature skip: out of BOTH sides of the rate below, which is + // computed as appearances - matureSkips - ungraded. + entry.ungraded += 1; } else if (isPassStatus(t.status)) { // lib/status.ts, not a raw "SUCCESS" literal: `status` is an // untyped string, and this page's pass rate must move with @@ -967,12 +991,13 @@ export function buildTagTaskRows(perRun: PerRun[], tag: string): TagTaskRow[] { // `?? true` only satisfies Map.get's `| undefined`; it is not a real // "unknown ⇒ keep" case.) if (!(newestTagged.get(taskId) ?? true)) continue; - const executed = e.appearances - e.matureSkips; + const executed = e.appearances - e.matureSkips - e.ungraded; rows.push({ taskId, skill: e.skill, appearances: e.appearances, matureSkips: e.matureSkips, + ungraded: e.ungraded, executed, passRate: executed > 0 ? (e.executedPasses / executed) * 100 : null, latestStatus: e.latestStatus, @@ -1092,9 +1117,10 @@ function rowFromScoped( ): RunListingRow { return { id, - tasksSucceeded: scoped.tasks.filter((t) => t.status === "SUCCESS") + tasksSucceeded: scoped.tasks.filter((t) => isPassStatus(t.status)) .length, tasksRun: scoped.tasks.length, + tasksGraded: scoped.tasks.filter((t) => isGraded(t.status)).length, totalCostUsd: scoped.totalCostUsd, taskDurationSeconds: scoped.taskDurationSeconds, harness: harness ?? null, @@ -1204,9 +1230,11 @@ export function buildAdhocRows( id, title: title ?? null, startedAt: overview.startedAt ?? null, - tasksSucceeded: overview.tasks.filter((t) => t.status === "SUCCESS") - .length, + tasksSucceeded: overview.tasks.filter((t) => + isPassStatus(t.status), + ).length, tasksRun: overview.tasks.length, + tasksGraded: overview.tasks.filter((t) => isGraded(t.status)).length, totalCostUsd: overview.totalCostUsd, taskDurationSeconds: overview.taskDurationSeconds, // Harness is a main-table-only (internal) column; ad-hoc rows omit it. diff --git a/evalboard/lib/pills.tsx b/evalboard/lib/pills.tsx index 1ab93e04..25cd715d 100644 --- a/evalboard/lib/pills.tsx +++ b/evalboard/lib/pills.tsx @@ -1,4 +1,4 @@ -import { statusCategory } from "./status"; +import { assertNever, statusCategory, type StatusCategory } from "./status"; // Shown on the green "Mature" status pill for a task that was skipped this run // (5 consecutive passes → re-validated only on its weekly slot) and carried @@ -51,6 +51,23 @@ export function MaturePill() { ); } +// A switch with an assertNever default, so a new StatusCategory fails +// `tsc --noEmit` here instead of quietly falling into whichever branch the +// boolean expression happened to leave it in. +function isFailureCategory(cat: StatusCategory): boolean { + switch (cat) { + case "failed": + case "error": + return true; + case "passed": + case "ungraded": + case "unknown": + return false; + default: + return assertNever(cat); + } +} + export function StatusPill({ status, relabel = false, @@ -66,7 +83,7 @@ export function StatusPill({ // Flow-execution failures (Faulted/Failed) land in statusCategory's "failed" // bucket too. Only null/unknown stays grey. const cat = statusCategory(status); - const isFailure = !ok && (cat === "failed" || cat === "error"); + const isFailure = !ok && isFailureCategory(cat); // Narrower list drives the relabel-to-"Failed" text so specific statuses // (e.g. MAX_TURNS_EXHAUSTED) keep their raw label while still showing red. const fail = @@ -82,14 +99,20 @@ export function StatusPill({ ? "bg-red-50 text-red-700 border-red-200" : "bg-gray-50 text-gray-600 border-gray-200"; const raw = status ?? "—"; + // NOT_GRADED gets a human label like every other status. Without it the grid + // rendered the raw enum token while its neighbours read "Passed"/"Failed" — + // and `isFailure` is already false for it, so the pill was correctly grey + // and incorrectly labelled. const label = relabel && ok ? "Passed" : relabel && status === "TIMEOUT" ? "Timed out" - : relabel && fail - ? "Failed" - : raw; + : relabel && cat === "ungraded" + ? "Not graded" + : relabel && fail + ? "Failed" + : raw; return ( (rows: readonly T[]): Map { const m = new Map(); for (const r of rows) { + // Ungraded replicates leave BOTH sides, exactly as they do in every + // per-row rate: they are excluded from the count AND from the map, so a + // task whose replicates were all ungraded does not appear at all rather + // than appearing as "0 of N passed". Without this an + // `execute --repeats 2` run renders a red "0/2 ✓". + if (!isGraded(r.status)) continue; const k = taskVariantKey(r); m.set(k, (m.get(k) ?? 0) + (isPassStatus(r.status) ? 1 : 0)); } return m; } +// Per (variant, task): how many replicates were GRADED. The denominator paired +// with perTaskPassCounts, so a badge reading "k/N ✓" never divides a graded +// numerator by an all-rows N. +export function perTaskGradedCounts< + T extends { taskId: string; variantId?: string | null; status: string | null }, +>(rows: readonly T[]): Map { + const m = new Map(); + for (const r of rows) { + if (!isGraded(r.status)) continue; + const k = taskVariantKey(r); + m.set(k, (m.get(k) ?? 0) + 1); + } + return m; +} + +// Compile-time exhaustiveness guard. Call it from a `switch`'s `default` arm +// over a StatusCategory: a new member then makes the argument non-`never` and +// `tsc --noEmit` fails at that site, which is what forces every consumer to be +// revisited instead of silently falling through an `else`. +export function assertNever(x: never): never { + throw new Error(`Unhandled status category: ${String(x)}`); +} + // Default table sort: failures and errors first, unknowns next, passes last. export function statusSortRank(status: string | null): number { const c = statusCategory(status); diff --git a/plugins/coder-eval/skills/analyze/SKILL.md b/plugins/coder-eval/skills/analyze/SKILL.md index 39e548ac..56b365b3 100644 --- a/plugins/coder-eval/skills/analyze/SKILL.md +++ b/plugins/coder-eval/skills/analyze/SKILL.md @@ -93,6 +93,17 @@ record is truncated or synthetic (the docker degrade path writes a `final_status for it. Note that `iterations` present but empty is a legitimate zero-turn record and not the same thing — test `has("iterations")`, never truthiness. +**A `final_status` of `NOT_GRADED` is not a failure — it is a row nothing measured.** +`coder-eval execute` runs the agent and deliberately skips every criterion, so such a +record has `weighted_score: null` and an EMPTY `success_criteria_results`. The recipe +above then reports `all_criteria_perfect: false` (the `length > 0` guard) with no +`failed_criteria`, which reads as "uniformly imperfect" for a run that was never scored. +Partition the rows first: exclude `NOT_GRADED` from BOTH sides of any pass rate or mean +score, count them separately, and say so in the report. If EVERY row is `NOT_GRADED`, +the answer is "this run was executed but not graded — grade it with `coder-eval run + --run-dir --resume` or `coder-eval evaluate ///00`", +not a table of zeros. + `error_excerpt` = the first ~200 characters of each failing criterion's `error`, falling back to `details`. Those are the only two free-text fields a criterion result carries, and which one is populated depends on the failure: `error` holds an exception, `details` diff --git a/pyproject.toml b/pyproject.toml index 5eea61f9..a5974a2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -226,6 +226,9 @@ external = [ "CE037", "CE038", "CE039", + "CE049", + "CE050", + "CE051", ] # custom architectural lint rules (tests/lint/) [tool.ruff.lint.pylint] diff --git a/src/coder_eval/cli/__init__.py b/src/coder_eval/cli/__init__.py index 0ded1cae..66eaddae 100644 --- a/src/coder_eval/cli/__init__.py +++ b/src/coder_eval/cli/__init__.py @@ -50,7 +50,7 @@ def main( - run: Execute evaluation tasks and grade them - execute: Execute evaluation tasks WITHOUT grading them - plan: Validate task files (dry-run) - - evaluate: Run criteria against a directory without an agent + - evaluate: Grade a directory against a task, or re-grade a finished run - report: Display or export evaluation reports - aggregate: Rebuild run.json/run.md from finalized task.json files """ diff --git a/src/coder_eval/cli/aggregate_command.py b/src/coder_eval/cli/aggregate_command.py index 9e3e1187..942c6f0e 100644 --- a/src/coder_eval/cli/aggregate_command.py +++ b/src/coder_eval/cli/aggregate_command.py @@ -78,11 +78,15 @@ def aggregate_command( skipped_tasks=skipped, ) write_run_summary(summary, out_dir) - console.print( - f"[green][OK][/green] Aggregated {summary.tasks_run} task(s) " - + f"({summary.tasks_succeeded} ok / {summary.tasks_failed} fail / {summary.tasks_error} err) " - + f"→ {out_dir / 'run.json'}" - ) + # The fourth bucket is named here too. `coder-eval aggregate ` is the + # step right after `coder-eval execute`, so an ungraded run is the FIRST + # thing this line renders — and without the term it reads + # "Aggregated 12 task(s) (0 ok / 0 fail / 0 err)", four numbers that no + # longer sum to tasks_run with nothing on screen to say where the rest went. + counts = f"{summary.tasks_succeeded} ok / {summary.tasks_failed} fail / {summary.tasks_error} err" + if summary.tasks_not_graded: + counts += f" / {summary.tasks_not_graded} not graded" + console.print(f"[green][OK][/green] Aggregated {summary.tasks_run} task(s) ({counts}) → {out_dir / 'run.json'}") console.print( "[dim]Note: run-level summary only — per-suite (suite.json/suite.md) and " + "experiment (experiment.json/experiment.md) rollups are not rebuilt.[/dim]" diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index d962b050..bcf10966 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -9,6 +9,7 @@ import typer +from ..evaluation.judge_persistence import TASK_JSON_TRANSCRIPT_EXCLUDE from ..logging_config import setup_logging from ..models import ( AgentKind, @@ -26,6 +27,7 @@ grading_sandbox_config, load_prior_result, regrade_in_place, + restore_pre_grade_record, task_from_prior, verify_reference_unchanged, ) @@ -34,7 +36,13 @@ from ..path_utils import PRE_GRADE_JSON_FILENAME, TASK_JSON_FILENAME, write_text_atomic from ..sandbox import Sandbox from .console import console -from .evaluate_target import EvaluateMode, EvaluateTarget, EvaluateTargetError, resolve_evaluate_target +from .evaluate_target import ( + EvaluateMode, + EvaluateTarget, + EvaluateTargetError, + as_work_dir, + resolve_evaluate_target, +) from .run_helpers import prepare_run_directory @@ -53,7 +61,14 @@ class _ResolvedInputs: prior: EvaluationResult | None -def _resolve_inputs(task_or_run_dir: Path, work_dir: Path | None, workspace: Path | None) -> _ResolvedInputs: +def _resolve_inputs( + task_or_run_dir: Path, + work_dir: Path | None, + workspace: Path | None, + *, + allow_recorded_commands: bool, + in_place: bool | None, +) -> _ResolvedInputs: """Turn the CLI positionals into a task, a workspace, and (maybe) a prior run. Split out of the command because it is where both shapes converge: after this @@ -71,23 +86,59 @@ def _resolve_inputs(task_or_run_dir: Path, work_dir: Path | None, workspace: Pat ) try: - return _resolve_run_dir_or_work_dir(target, workspace) + return _resolve_run_dir_or_work_dir( + target, workspace, allow_recorded_commands=allow_recorded_commands, in_place=in_place + ) except RegradeError as e: # The shared core raises a plain exception (orchestration/ must not # depend on the CLI layer, CE004); surface it as a CLI error here. raise typer.BadParameter(str(e)) from e -def _resolve_run_dir_or_work_dir(target: EvaluateTarget, workspace: Path | None) -> _ResolvedInputs: +def _resolve_run_dir_or_work_dir( + target: EvaluateTarget, + workspace: Path | None, + *, + allow_recorded_commands: bool, + in_place: bool | None, +) -> _ResolvedInputs: """The mode-specific half of :func:`_resolve_inputs`.""" prior: EvaluationResult | None = None + if target.mode is EvaluateMode.RUN_DIR and target.task_file is not None: + # `is_run_dir` is a filename probe, so a plain work directory holding an + # unrelated file called task.json lands here. That would abort the + # pre-existing `evaluate ` form on a pydantic wall the + # user can only escape by renaming their own file. The task file is + # already in hand, so fall back to the shape they asked for. + try: + load_prior_result(target.target) + except RegradeError as e: + logger.warning( + "%s holds a %s that is not a readable run record (%s); grading it as a plain " + "work directory.", + target.target, + TASK_JSON_FILENAME, + e, + ) + target = as_work_dir(target) + if target.mode is EvaluateMode.RUN_DIR: prior = load_prior_result(target.target) if target.task_file is not None: task, source_yaml = load_task(target.task_file) console.print(f"[dim]Grading with {target.task_file} (overrides the run's recorded config).[/dim]") else: - task, source_yaml = task_from_prior(prior, target.target) + # pre_run/post_run are skipped on the in-place path (an adopted + # workspace must not have its hooks re-run over the agent's + # deliverables), so there they are not shell the run dir can make + # this host execute — only --copy re-runs them. Mirrors the + # grade_in_place default computed in run_evaluation. + hooks_will_run = not (in_place if in_place is not None else True) + task, source_yaml = task_from_prior( + prior, + target.target, + allow_recorded_commands=allow_recorded_commands, + include_hooks=hooks_will_run, + ) work_dir = workspace or default_workspace(target.target, prior) recorded_source = prior.task_config.source_file if prior.task_config else None task_file = target.task_file or (Path(recorded_source) if recorded_source else None) @@ -195,6 +246,24 @@ def evaluate_command( "an adopted directory is never moved or deleted." ), ), + allow_recorded_commands: bool = typer.Option( + False, + "--allow-recorded-commands", + help=( + "Accept shell commands (run_command criteria, pre_run/post_run) rebuilt from the run " + "directory's own task.json. A run directory is a shareable artifact, so its recorded " + "config is untrusted input; without this, grading refuses rather than running it here." + ), + ), + allow_host_grading: bool = typer.Option( + False, + "--allow-host-grading", + help=( + "Grade a `driver: docker` run on this host. Grading cannot start a container, so the " + "criteria run against a filesystem that lacks the container's paths and toolchain — " + "scores may differ from the run. Such rows are stamped graded_on_host." + ), + ), run_dir: Path | None = typer.Option( # noqa: B008 None, "--run-dir", @@ -229,6 +298,8 @@ def evaluate_command( in_place=in_place, verbose=verbose, preserve=preserve, + allow_recorded_commands=allow_recorded_commands, + allow_host_grading=allow_host_grading, run_dir=run_dir, ) @@ -241,6 +312,8 @@ def run_evaluation( in_place: bool | None = None, verbose: bool = False, preserve: bool = True, + allow_recorded_commands: bool = False, + allow_host_grading: bool = False, run_dir: Path | None = None, ) -> None: """The body of ``coder-eval evaluate``, with real Python defaults. @@ -254,7 +327,13 @@ def run_evaluation( console.print("\n[bold]Evaluating Criteria[/bold]\n") - inputs = _resolve_inputs(task_or_run_dir, work_dir, workspace) + inputs = _resolve_inputs( + task_or_run_dir, + work_dir, + workspace, + allow_recorded_commands=allow_recorded_commands, + in_place=in_place, + ) task = inputs.task source_yaml = inputs.source_yaml graded_dir = inputs.work_dir @@ -274,7 +353,11 @@ def run_evaluation( console.print(f"[red]✗ Failed to prepare run directory:[/red] {e}") raise typer.Exit(1) from e - sandbox_config = grading_sandbox_config(task) + try: + sandbox_config = grading_sandbox_config(task, allow_host_grading=allow_host_grading) + except RegradeError as e: + console.print(f"[red]✗ {e}[/red]") + raise typer.Exit(1) from e if not grade_in_place: # Copy path: preload the sandbox with the work dir as a template source. template_source = TemplateDirSource(path=str(graded_dir.resolve())) @@ -299,6 +382,7 @@ async def _setup_and_run() -> EvaluationResult: source_yaml=source_yaml, variant_id=prior.variant_id, replicate_index=_replicate_index_of(target.target), + allow_host_grading=allow_host_grading, ) if grade_in_place: await asyncio.to_thread(sandbox.adopt, graded_dir) @@ -323,6 +407,27 @@ async def _setup_and_run() -> EvaluationResult: result = asyncio.run(_setup_and_run()) + # BEFORE the count guard below. A grading crash returns a populated ERROR + # result with an EMPTY criteria list (Orchestrator.run() converts internal + # failures into a result rather than raising), so the count check fires + # first and the user is told only "Result count mismatch: got 0, expected 2" + # — the real error is never printed, and the "still re-gradeable" notice is + # unreachable on exactly the path it was written for. + if result.final_status is FinalStatus.ERROR: + console.print(f"\n[red]✗ Evaluation error: {result.error_message}[/red]") + if prior is not None: + # A grading-time crash (a failing checker, an unreachable judge) is + # not a verdict about the run. Leaving ERROR on disk would replace a + # perfectly re-gradeable NOT_GRADED row with one BOTH commands treat + # as permanently complete, so the run could never be graded again + # without hand-restoring task.execute.json. + restore_pre_grade_record(target.target) + console.print( + f"[yellow]⚠[/] Grading errored; {target.target / TASK_JSON_FILENAME} is left " + + "ungraded so the run stays re-gradeable." + ) + raise typer.Exit(1) + # Display results console.print("[bold]Criteria Results:[/bold]\n") @@ -371,24 +476,9 @@ async def _setup_and_run() -> EvaluationResult: f"[dim]Re-graded {prior.final_status.value} → {result.final_status.value} " + f"over {len(result.iterations)} recorded turn(s).[/dim]" ) - if result.final_status is FinalStatus.ERROR: - # A grading-time crash (a failing checker, an unreachable judge) is - # not a verdict about the run. Writing it back would replace a - # perfectly re-gradeable NOT_GRADED row with ERROR — which BOTH - # commands treat as permanently complete, so the run could never be - # graded again without hand-restoring task.execute.json. The - # diagnostic row is still in this grade's own run dir. - console.print( - f"[yellow]⚠[/] Grading errored; leaving {target.target / TASK_JSON_FILENAME} " - + "as it was so the run stays re-gradeable." - ) - else: - _write_back(target.target, result) + _write_back(target.target, result) - if result.final_status == FinalStatus.ERROR: - console.print(f"\n[red]✗ Evaluation error: {result.error_message}[/red]") - raise typer.Exit(1) - elif failed == 0: + if failed == 0: console.print("\n[green]All criteria passed! ✓[/green]") raise typer.Exit(0) else: @@ -419,7 +509,7 @@ def _write_back(run_dir: Path, result: EvaluationResult) -> None: # Atomic, matching the orchestrator's own task.json writer: a torn write # here makes the row parse as malformed, which a later --resume reads as # "not complete" and re-pays for the agent. - write_text_atomic(target, result.model_dump_json(indent=2)) + write_text_atomic(target, result.model_dump_json(indent=2, exclude=TASK_JSON_TRANSCRIPT_EXCLUDE)) except OSError as e: # Never fail the grade over the write-back: the verdict was computed and # already printed, and the fresh run dir holds its own task.json. diff --git a/src/coder_eval/cli/evaluate_target.py b/src/coder_eval/cli/evaluate_target.py index f9b72b04..b16e8857 100644 --- a/src/coder_eval/cli/evaluate_target.py +++ b/src/coder_eval/cli/evaluate_target.py @@ -94,5 +94,27 @@ def resolve_evaluate_target(first: Path, second: Path | None) -> EvaluateTarget: # criteria against an expensive run I already paid for" case, which is the # main reason to keep `execute` and `evaluate` separate at all. Allow it, and # let the caller be told which config won. + # + # This probe is a filename test, so a plain work directory that merely + # happens to contain a file called `task.json` is read as a run directory — + # and the pre-existing two-argument form would abort on a pydantic wall with + # no way to override it. It is not repaired here (this function is pure and + # cannot tell a real record from a namesake); the caller re-reads the record + # and falls back to WORK_DIR when it does not parse. See + # ``evaluate_command._resolve_run_dir_or_work_dir``. mode = EvaluateMode.RUN_DIR if second.is_dir() and is_run_dir(second) else EvaluateMode.WORK_DIR return EvaluateTarget(mode=mode, target=second, task_file=first) + + +def as_work_dir(target: EvaluateTarget) -> EvaluateTarget: + """Re-read a two-argument target as the plain work-directory shape. + + The escape hatch for the namesake ``task.json`` above. Only valid when a task + file was supplied, which is exactly the two-argument form. + """ + if target.task_file is None: + raise EvaluateTargetError( + f"{target.target} holds a {TASK_JSON_FILENAME} that is not a readable run record, and no " + + f"task file was given. Pass one: coder-eval evaluate {target.target}" + ) + return EvaluateTarget(mode=EvaluateMode.WORK_DIR, target=target.target, task_file=target.task_file) diff --git a/src/coder_eval/cli/execute_command.py b/src/coder_eval/cli/execute_command.py index 79388003..15bbdc2a 100644 --- a/src/coder_eval/cli/execute_command.py +++ b/src/coder_eval/cli/execute_command.py @@ -196,8 +196,10 @@ def execute_command( ERROR / TIMEOUT / TOKEN_BUDGET_EXCEEDED and exits non-zero exactly as under `run`. Only the verdict is withheld, never the facts of the run. - Not supported here: --junit-xml (no verdicts to report) and simulation tasks - (their turn-continuation logic reads criteria results). + Not supported here: --junit-xml (no verdicts to report), --allow-host-grading + (nothing is graded, so there is no host-grading decision to make — pass it to + `coder-eval run --resume` or `coder-eval evaluate` instead), and simulation + tasks (their turn-continuation logic reads criteria results). Examples: @@ -213,6 +215,8 @@ def execute_command( resume=resume, # Not exposed as a flag — a JUnit report reports verdicts, and there are none. junit_xml=None, + # Same reason: nothing is graded here, so there is no host-grading choice. + allow_host_grading=False, max_parallel=max_parallel, verbose=verbose, log_file=log_file, diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index be6ee073..336370cd 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -8,6 +8,7 @@ import urllib.parse import urllib.request from collections.abc import Callable +from datetime import datetime from pathlib import Path from typing import Any @@ -17,7 +18,15 @@ from ..config import Settings, settings from ..logging_config import setup_logging -from ..models import EvaluationResult, FinalStatus, PreservationMode, ResolvedTask, RunSummary, TaskResult +from ..models import ( + AgentKind, + EvaluationResult, + FinalStatus, + PreservationMode, + ResolvedTask, + RunSummary, + TaskResult, +) from ..orchestration.config import BatchRunConfig from ..path_utils import create_latest_symlink, format_task_log_id from ..streaming.callbacks import CompositeStreamCallback @@ -204,6 +213,17 @@ def run_command( "set and pays for the agent twice." ), ), + allow_host_grading: bool = typer.Option( + False, + "--allow-host-grading", + help=( + "When --resume grades a `driver: docker` row, grade it on this host anyway. " + "Grading cannot start a container, so such criteria run against a filesystem " + "lacking the container's paths and toolchain and may score differently than " + "the run did; those rows are stamped graded_on_host. Without this they are " + "refused and stay ungraded." + ), + ), max_parallel: int = typer.Option( 1, "--max-parallel", @@ -366,6 +386,7 @@ def run_command( preservation_mode=preservation_mode, run_dir=run_dir, resume=resume, + allow_host_grading=allow_host_grading, max_parallel=max_parallel, verbose=verbose, log_file=log_file, @@ -393,6 +414,7 @@ def run_pipeline( preservation_mode: PreservationMode | None, run_dir: Path | None, resume: bool, + allow_host_grading: bool, max_parallel: int, verbose: bool, log_file: Path | None, @@ -477,6 +499,7 @@ def run_pipeline( repeats=repeats, verbose=verbose, resume=resume, + allow_host_grading=allow_host_grading, include_skipped=include_skipped, junit_xml=junit_xml, grade=grade, @@ -503,6 +526,7 @@ async def _run_all_tasks( repeats: int | None = None, verbose: bool = False, resume: bool = False, + allow_host_grading: bool = False, include_skipped: bool = False, junit_xml: Path | None = None, grade: bool = True, @@ -583,7 +607,14 @@ async def _run_all_tasks( try: # Always run through experiment layer (defaults to experiments/default.yaml) summary, failed_suite_gates = await _run_with_experiment( - all_task_files, config, experiment_path, stream_mode, max_parallel, resume=resume, grade=grade + all_task_files, + config, + experiment_path, + stream_mode, + max_parallel, + resume=resume, + grade=grade, + allow_host_grading=allow_host_grading, ) # Aggregate task logs into run.log @@ -670,7 +701,28 @@ def _on_task_complete(result: Any) -> None: return result -async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[ResolvedTask, TaskResult]]: +def _unreadable_row_placeholder(rt: ResolvedTask, error: Exception) -> EvaluationResult: + """A minimal ungraded row for a task whose recorded result cannot be read. + + Exists so an unreadable row stays IN ``run.json`` and in + ``tasks_not_graded`` — the counter the exit gate reads. Dropping it silently + made a resume that graded nothing exit 0. + """ + return EvaluationResult( + task_id=rt.task.task_id, + task_description=rt.task.description, + variant_id=rt.variant_id, + agent_type=str(rt.task.agent.type) if rt.task.agent and rt.task.agent.type else AgentKind.NONE.value, + started_at=datetime.now(), + final_status=FinalStatus.NOT_GRADED, + iteration_count=0, + error_message=f"Grading failed during --resume: the recorded result could not be read ({error})", + ) + + +async def _grade_resumed_tasks( + to_grade: list[ResolvedTask], *, allow_host_grading: bool = False +) -> list[tuple[ResolvedTask, TaskResult]]: """Grade the rows ``coder-eval execute`` left NOT_GRADED, in place. Each task's trajectory and workspace are already on disk, so this runs the @@ -700,10 +752,10 @@ async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[Resol default_workspace, load_prior_result, regrade_in_place, + restore_pre_grade_record, ) graded: list[tuple[ResolvedTask, TaskResult]] = [] - failed_to_load: list[str] = [] for rt in to_grade: # Inside the try: an unreadable row must skip like any other grading # failure. Outside it, one bad task.json propagates out of the loop and @@ -727,22 +779,25 @@ async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[Resol source_yaml=rt.source_yaml, variant_id=rt.variant_id, replicate_index=rt.replicate_index, + allow_host_grading=allow_host_grading, ) except (RegradeError, OSError, RuntimeError, ValueError) as e: console.print(f"[yellow]⚠[/] Could not grade {rt.task.task_id}: {e}") if prior is None: - # The row could not even be read, so there is nothing to fold - # back. Skipping keeps it out of run.json exactly as it already - # is on disk, and the exit gate still fails the command because - # a task the resume owed a grade produced none. - failed_to_load.append(rt.task.task_id) - continue - # Stamp the reason onto the row. Without it the failure survives only - # in this console line: the folded-back result keeps the execute - # phase's empty error_message, so run.json, the reports and CI show - # an ungraded row with no explanation of why grading never happened. - result = prior - result.error_message = f"Grading failed during --resume: {e}" + # The row could not even be read, so there is no recorded result + # to fold back — but dropping it entirely removes it from + # run.json AND from `tasks_not_graded`, which is what the exit + # gate counts, so a resume whose rows were all unreadable would + # report success. Stand in a minimal ungraded row instead: it + # keeps the task visible and keeps the command non-zero. + result = _unreadable_row_placeholder(rt, e) + else: + # Stamp the reason onto the row. Without it the failure survives + # only in this console line: the folded-back result keeps the + # execute phase's empty error_message, so run.json, the reports + # and CI show an ungraded row with no explanation. + result = prior + result.error_message = f"Grading failed during --resume: {e}" else: if result.final_status is FinalStatus.ERROR: # An orchestrator-level grading crash is not a verdict about the @@ -751,6 +806,13 @@ async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[Resol # `except` above never sees them and the ERROR row replaces a # perfectly re-gradeable NOT_GRADED one — and ERROR is "complete" # for both commands, so the row could never be graded again. + # + # Fixing the in-memory result is only half of it: _finalize_result + # already wrote the ERROR task.json into this same directory + # before returning, so run.json would say NOT_GRADED while the + # row on disk says ERROR — and the on-disk one is what a later + # --resume reads. Put the pre-grade record back. + restore_pre_grade_record(rt.run_dir) console.print( f"[yellow]⚠[/] Grading {rt.task.task_id} errored ({result.error_message}); " + "keeping the ungraded row so it stays re-gradeable." @@ -774,6 +836,44 @@ async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[Resol return graded +async def _apply_resume( + resolved: list[ResolvedTask], *, grade: bool, allow_host_grading: bool +) -> tuple[list[ResolvedTask], list[TaskResult], list[ResolvedTask]]: + """Split a resumed run into what still needs running, and what is carried in. + + Extracted from ``_run_with_experiment``, which answers several questions at + once; this one answers "what does the resume still owe?" and is where the + grading half of the answer lives. + + Returns ``(to_run, prior_results, prior_resolved)``. ``resolved`` itself is + NOT narrowed — the suite rollups downstream need every task, run or not. + """ + from ..orchestration.batch import clear_rerun_artifacts, partition_for_resume + + part = partition_for_resume(resolved, grade=grade) + prior_results = list(part.prior_results) + prior_resolved = list(part.prior_resolved) + # A re-run task re-executes from scratch, so any leftover artifacts (only + # DIRECT_WRITE writes them live; a container killed mid-run leaves partials) + # are stale and could let a file-based criterion pass on the old output. + # to_grade is deliberately NOT cleared: its artifacts are the run's output + # and the very thing being graded. + cleared = clear_rerun_artifacts(part.to_run) + console.print( + f"[cyan]↻ Resume:[/] {len(prior_results)} task(s) already complete, " + + f"running {len(part.to_run)} remaining" + + (f", grading {len(part.to_grade)} executed-but-ungraded" if part.to_grade else "") + + (f" (cleared {cleared} stale artifact dir(s))" if cleared else "") + ) + # Grade the rows `execute` left behind, reusing the trajectory and workspace + # already on disk rather than paying for the agent twice. Folded in as + # prior_results so the summary covers them like any other. + for rt, tr in await _grade_resumed_tasks(part.to_grade, allow_host_grading=allow_host_grading): + prior_results.append(tr) + prior_resolved.append(rt) + return part.to_run, prior_results, prior_resolved + + async def _run_with_experiment( all_task_files: list[Path], config: BatchRunConfig, @@ -782,6 +882,7 @@ async def _run_with_experiment( max_parallel: int, resume: bool = False, grade: bool = True, + allow_host_grading: bool = False, ) -> tuple[RunSummary, int]: """Run tasks through the experiment resolution layer. @@ -801,10 +902,8 @@ async def _run_with_experiment( RunSummary with aggregated results. """ from ..orchestration.batch import ( - clear_rerun_artifacts, compute_run_fingerprint, fingerprint_diff, - partition_for_resume, read_run_fingerprint, run_batch, write_run_fingerprint, @@ -904,26 +1003,9 @@ async def _run_with_experiment( prior_results: list[TaskResult] = [] prior_resolved: list[ResolvedTask] = [] if resume: - part = partition_for_resume(resolved, grade=grade) - to_run, prior_results, prior_resolved = part.to_run, part.prior_results, part.prior_resolved - # A re-run task re-executes from scratch, so any leftover artifacts (only - # DIRECT_WRITE writes them live; a container killed mid-run leaves partials) - # are stale and could let a file-based criterion pass on the old output. - # to_grade is deliberately NOT cleared: its artifacts are the run's output - # and the very thing being graded. - cleared = clear_rerun_artifacts(to_run) - console.print( - f"[cyan]↻ Resume:[/] {len(prior_results)} task(s) already complete, " - + f"running {len(to_run)} remaining" - + (f", grading {len(part.to_grade)} executed-but-ungraded" if part.to_grade else "") - + (f" (cleared {cleared} stale artifact dir(s))" if cleared else "") + to_run, prior_results, prior_resolved = await _apply_resume( + resolved, grade=grade, allow_host_grading=allow_host_grading ) - # Grade the rows `execute` left behind, reusing the trajectory and - # workspace already on disk rather than paying for the agent twice. - # Folded in as prior_results so the summary covers them like any other. - for rt, tr in await _grade_resumed_tasks(part.to_grade): - prior_results.append(tr) - prior_resolved.append(rt) # Print execution mode print_execution_mode(len(to_run), max_parallel) diff --git a/src/coder_eval/cli/run_task_internal_command.py b/src/coder_eval/cli/run_task_internal_command.py index e3c4ad85..62f176fe 100644 --- a/src/coder_eval/cli/run_task_internal_command.py +++ b/src/coder_eval/cli/run_task_internal_command.py @@ -191,7 +191,11 @@ def _watch_host_heartbeat() -> None: # We're already inside the container; another nested docker would be # both wrong and impossible (no docker CLI in image). if task.sandbox.driver == "docker": - task = task.model_copy(update={"sandbox": task.sandbox.model_copy(update={"driver": "tempdir"})}) + # noqa: CE051 — the ONE legitimate rewrite. We are already inside the + # container the docker driver asked for, so the isolation the driver + # names is present, not bypassed; a nested docker would be both wrong + # and impossible (no docker CLI in the image). + task = task.model_copy(update={"sandbox": task.sandbox.model_copy(update={"driver": "tempdir"})}) # noqa: CE051 output_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 770a1bb7..43b1278f 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -890,8 +890,17 @@ class SuiteRollup(BaseModel): # The fourth bucket, matching RunSummary.tasks_not_graded and # VariantAggregate.tasks_not_graded. Defaulted so a suite.json written before # `execute` existed still parses. - rows_not_graded: int = 0 - pass_rate: float = Field(ge=0.0, le=1.0, description="rows_passed / rows_graded (ungraded rows excluded)") + rows_not_graded: int = Field(default=0, ge=0) + pass_rate: float | None = Field( + default=None, + ge=0.0, + le=1.0, + description=( + "rows_passed / rows_graded (ungraded rows excluded). None — never 0.0 — when nothing " + "was graded: a suite that was never measured has no pass rate, and 0.0 would publish " + "'0.0%' for it, indistinguishable from a suite where every row failed." + ), + ) average_weighted_score: float | None = Field( default=None, description="Mean weighted_score across rows that produced one." ) @@ -915,6 +924,18 @@ class SuiteRollup(BaseModel): ), ) + @computed_field # type: ignore[prop-decorator] + @property + def rows_graded(self) -> int: + """The denominator behind ``pass_rate``, serialized like its two twins. + + ``RunSummary.tasks_graded`` exists for the same reason and states it: a + consumer that cannot read the denominator has to re-derive it, which is + precisely how two surfaces end up publishing different numbers for the + same suite. + """ + return self.rows_total - self.rows_not_graded + @model_validator(mode="after") def _check_row_count_invariant(self) -> SuiteRollup: """The same guard RunSummary carries, which this model was missing. diff --git a/src/coder_eval/models/tasks.py b/src/coder_eval/models/tasks.py index 68cfd324..52a23144 100644 --- a/src/coder_eval/models/tasks.py +++ b/src/coder_eval/models/tasks.py @@ -733,7 +733,10 @@ def check_suite_thresholds_require_dataset(self) -> Self: """ if self.dataset is None and self.suite_id is None: for c in self.success_criteria: - if getattr(c, "suite_thresholds", None): + # Direct attribute access, not getattr: suite_thresholds is + # declared on BaseSuccessCriterion, so every union member has it + # and pyright can see a rename. + if c.suite_thresholds: raise ValueError( f"success_criteria[{c.type!r}].suite_thresholds requires a dataset: block " + "(thresholds are evaluated on aggregated across-row metrics)" diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index 58cc3558..6c19ea14 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -11,6 +11,7 @@ import importlib.resources import logging import re +from collections.abc import Sequence from pathlib import Path from typing import Any @@ -822,17 +823,46 @@ def _pick_worst_status(statuses: list[FinalStatus]) -> FinalStatus: "ungraded" sorts LEAST urgent (above "succeeded") — it carries no verdict, so any replicate that does have one must win. It therefore survives only when - every replicate is ungraded, which is the only case reachable today anyway - (``grade`` is run-level, so replicates never mix). + every replicate is ungraded. + + Mixed sets ARE reachable, contrary to an earlier note here: ``grade`` is + run-level, but ``run --resume`` grades each executed-but-ungraded row + independently and folds a row whose grade failed back in still ungraded + (``cli/run_command._grade_resumed_tasks``). So one replicate can be graded + while its sibling is not, and the ordering above is what keeps that from + absorbing an unmeasured replicate into a pass. """ priority = {"error": 0, "failed": 1, "succeeded": 2, "ungraded": 3} return min(statuses, key=lambda s: priority.get(s.category, -1)) +def _measured_scores(rows: Sequence[VariantResult] | Sequence[TaskResult]) -> list[float]: + """The scores of every row that was actually GRADED, errors included as 0.0. + + The filter is on ``final_status.category``, deliberately not on + ``weighted_score is not None``. Those look equivalent and are not: an + ERROR / BUILD_FAILED row also has no score, and dropping it removes it from + BOTH sides of the mean. A nightly where one image build failed would then + report a HIGHER headline score than a clean one, and A/B comparisons would + be biased toward whichever variant errored more — the exact "bonus for + erroring" the run-level rates are written to avoid. + + Only ``ungraded`` rows leave both sides: nothing measured them, so they are + not a miss. Everything else that ran and produced no score IS a miss. + """ + scores: list[float] = [] + for row in rows: + result = row if isinstance(row, VariantResult) else row.result + if result.final_status.category == "ungraded": + continue + scores.append(result.weighted_score if result.weighted_score is not None else 0.0) + return scores + + def _mean_graded_score(vr_list: list[VariantResult]) -> float | None: - """Mean ``weighted_score`` over the graded rows; ``None`` when none were graded.""" - graded = [v.weighted_score for v in vr_list if v.weighted_score is not None] - return sum(graded) / len(graded) if graded else None + """Mean score over the measured rows; ``None`` when none were measured.""" + scores = _measured_scores(vr_list) + return sum(scores) / len(scores) if scores else None def _mean_reference_similarity(reps: list[TaskResult]) -> float | None: @@ -846,6 +876,37 @@ def _mean_reference_similarity(reps: list[TaskResult]) -> float | None: return sum(scores) / len(scores) if scores else None +def _fold_replicates(task_id: str, variant_id: str, reps: list[TaskResult]) -> VariantResult: + """Fold every replicate of one (task, variant) into a single VariantResult. + + Extracted from ``aggregate_results``, which was already the largest function + in the module before the ungraded bucket added another filter to it. + """ + # Ungraded replicates — and ONLY those — drop out entirely rather than + # contributing 0.0: `or 0.0` would average a clean `execute` run down to a + # real-looking zero, while dropping an errored one would pay it a bonus. + scores = _measured_scores(reps) + non_errored = [r for r in reps if r.result.final_status.category != "error"] + durations = [r.result.duration_seconds for r in non_errored] + iter_counts = [r.result.iteration_count for r in reps if r.result.iteration_count is not None] + asst_turns = [r.result.total_assistant_turns for r in reps if r.result.total_assistant_turns is not None] + token_vals = [r.result.total_token_usage.total_tokens for r in reps if r.result.total_token_usage is not None] + + return VariantResult( + variant_id=variant_id, + task_id=task_id, + weighted_score=sum(scores) / len(scores) if scores else None, + final_status=_pick_worst_status([r.result.final_status for r in reps]), + duration_seconds=sum(durations), + total_tokens=sum(token_vals) if token_vals else None, + iteration_count=round(sum(iter_counts) / len(iter_counts)) if iter_counts else None, + total_assistant_turns=round(sum(asst_turns) / len(asst_turns)) if asst_turns else None, + reference_similarity=_mean_reference_similarity(reps), + replicate_index=0, # aggregate — points at first replicate for link rendering + replicate_count=len(reps), + ) + + def aggregate_results( experiment_id: str, description: str, @@ -878,48 +939,32 @@ def aggregate_results( # Collect per-replicate scores keyed variant_id → task_id → [scores] for stats rendering. per_replicate_scores: dict[str, dict[str, list[float]]] = {} for (task_id, variant_id), reps in task_variant_reps.items(): - per_replicate_scores.setdefault(variant_id, {})[task_id] = [ - r.result.weighted_score for r in reps if r.result.weighted_score is not None - ] + per_replicate_scores.setdefault(variant_id, {})[task_id] = _measured_scores(reps) task_variants: dict[str, list[VariantResult]] = {} for (task_id, variant_id), reps in task_variant_reps.items(): - # Ungraded replicates drop out entirely rather than contributing 0.0 — - # `or 0.0` would average a clean `execute` run down to a real-looking zero. - scores = [r.result.weighted_score for r in reps if r.result.weighted_score is not None] - non_errored = [r for r in reps if r.result.final_status.category != "error"] - durations = [r.result.duration_seconds for r in non_errored] - statuses = [r.result.final_status for r in reps] - iter_counts = [r.result.iteration_count for r in reps if r.result.iteration_count is not None] - asst_turns = [r.result.total_assistant_turns for r in reps if r.result.total_assistant_turns is not None] - token_vals = [r.result.total_token_usage.total_tokens for r in reps if r.result.total_token_usage is not None] - ref_similarity = _mean_reference_similarity(reps) - final_status = _pick_worst_status(statuses) - - variant_result = VariantResult( - variant_id=variant_id, - task_id=task_id, - weighted_score=sum(scores) / len(scores) if scores else None, - final_status=final_status, - duration_seconds=sum(durations), - total_tokens=sum(token_vals) if token_vals else None, - iteration_count=round(sum(iter_counts) / len(iter_counts)) if iter_counts else None, - total_assistant_turns=round(sum(asst_turns) / len(asst_turns)) if asst_turns else None, - reference_similarity=ref_similarity, - replicate_index=0, # aggregate — points at first replicate for link rendering - replicate_count=len(reps), - ) - task_variants.setdefault(task_id, []).append(variant_result) + task_variants.setdefault(task_id, []).append(_fold_replicates(task_id, variant_id, reps)) # Build task summaries task_summaries: list[TaskExperimentSummary] = [] for task_id, variants in task_variants.items(): # Only graded variants can win or set a spread. Including ungraded ones # at 0.0 would name an arbitrary "best" among scores that do not exist. + # + # When NOTHING was scored there is no winner, and the fallback must not + # invent one: `variants[0]` is whichever arm the input happened to list + # first, so swapping the two inputs flipped the reported winner — with + # `is_tie=False` asserting it was a real result. Sort by variant_id (so + # the field is at least deterministic) and mark it a tie among all arms, + # which is what "no arm outscored another" actually means. scored = [(v, v.weighted_score) for v in variants if v.weighted_score is not None] - best = max(scored, key=lambda pair: (pair[1], pair[0].variant_id))[0] if scored else variants[0] + if scored: + best = max(scored, key=lambda pair: (pair[1], pair[0].variant_id))[0] + top_count = sum(1 for _, s in scored if s == best.weighted_score) + else: + best = min(variants, key=lambda v: v.variant_id) + top_count = len(variants) scores = [s for _, s in scored] - top_count = sum(1 for _, s in scored if s == best.weighted_score) rep_counts = {v.replicate_count for v in variants} task_summaries.append( TaskExperimentSummary( diff --git a/src/coder_eval/orchestration/regrade.py b/src/coder_eval/orchestration/regrade.py index 892cee0e..54f598d1 100644 --- a/src/coder_eval/orchestration/regrade.py +++ b/src/coder_eval/orchestration/regrade.py @@ -27,7 +27,7 @@ TaskConfigRecord, TaskDefinition, ) -from coder_eval.path_utils import PRE_GRADE_JSON_FILENAME, TASK_JSON_FILENAME +from coder_eval.path_utils import PRE_GRADE_JSON_FILENAME, TASK_JSON_FILENAME, write_text_atomic from coder_eval.sandbox import Sandbox @@ -53,7 +53,13 @@ def load_prior_result(run_dir: Path) -> EvaluationResult: raise RegradeError(f"{path} is not a readable EvaluationResult: {e}") from e -def task_from_prior(prior: EvaluationResult, run_dir: Path) -> tuple[TaskDefinition, str]: +def task_from_prior( + prior: EvaluationResult, + run_dir: Path, + *, + allow_recorded_commands: bool = False, + include_hooks: bool = True, +) -> tuple[TaskDefinition, str]: """Rebuild the executed task from the run's own recorded config. Rebuilding from ``task_config.resolved`` rather than re-reading the YAML is @@ -65,6 +71,9 @@ def task_from_prior(prior: EvaluationResult, run_dir: Path) -> tuple[TaskDefinit Falls back to the source YAML only when ``resolved`` will not validate (a schema change since the run), and says so loudly — a quiet fallback would reintroduce exactly the drift above. + + ``allow_recorded_commands`` gates the shell half. See + :func:`check_embedded_commands`. """ record = prior.task_config if record is None: @@ -75,35 +84,98 @@ def task_from_prior(prior: EvaluationResult, run_dir: Path) -> tuple[TaskDefinit try: task = TaskDefinition.model_validate(record.resolved) except ValueError as e: - return _fall_back_to_source(record, run_dir, e) - warn_on_embedded_commands(task, run_dir) + return _fall_back_to_source( + record, run_dir, e, allow_recorded_commands=allow_recorded_commands, include_hooks=include_hooks + ) + check_embedded_commands(task, run_dir, allow_recorded_commands=allow_recorded_commands, include_hooks=include_hooks) return task, record.source_yaml -def warn_on_embedded_commands(task: TaskDefinition, run_dir: Path) -> None: - """Name the shell commands a rebuilt config will execute on this host. +def embedded_commands(task: TaskDefinition, *, include_hooks: bool = True) -> list[str]: + """Every shell command a rebuilt task definition would run on this host. + + ``include_hooks`` covers ``pre_run`` / ``post_run``. They are SKIPPED on the + in-place grading path (``Sandbox.was_adopted`` — re-running them would + overwrite the agent's deliverables before the criteria read them), so on that + path they are not a capability the run dir has; on the ``--copy`` path they + are. + + ``isinstance`` narrowing, never ``getattr(c, "command", None)``: an untyped + string probe over a discriminated union is invisible to pyright, so renaming + a field silently degrades the only guard on this path to a permanent no-op — + the exact hazard ``models/tasks.py`` already documents in prose. It also + cannot reach ``agent_judge``, whose ``bash`` tooling is the widest blast + radius of the three. + """ + from coder_eval.models import AgentJudgeCriterion, RunCommandCriterion, UiPathEvalCriterion + + commands: list[str] = [] + for c in task.success_criteria: + if isinstance(c, RunCommandCriterion): + commands.append(c.command) + elif isinstance(c, AgentJudgeCriterion): + # No command string of its own: it spawns a Claude Code SDK agent + # with tool access (Bash included) under the grader's credentials, + # which is a strictly wider capability than one shell line. + commands.append(f"") + elif isinstance(c, UiPathEvalCriterion): + # Builds and shells `uv run uipath eval …`. Every argument is + # shlex-quoted, so this is disclosure rather than injection — but it + # is still a subprocess the recorded config chose to start. + commands.append(f"uv run uipath eval {c.agent_name} {c.eval_set}") + if include_hooks: + commands += [c.command for c in task.pre_run] + [c.command for c in task.post_run] + return commands + + +def check_embedded_commands( + task: TaskDefinition, run_dir: Path, *, allow_recorded_commands: bool, include_hooks: bool = True +) -> None: + """Refuse — or at minimum name — the shell a rebuilt config will run here. ``task_config.resolved`` is data that travels inside a run directory, and a run directory is a shareable artifact — the detached-grading flow exists so one machine can execute and another can grade. Rebuilding the task from it means the *run dir* decides what ``run_command`` criteria the grader runs, - with the grader's environment. That is the intended behavior (it is how the - grade reproduces the executed config), but it must not be invisible: print - what will run so an unexpected command is noticed before it executes. + with the grader's environment (API keys, cloud credentials, SSH agent). + + A warning is not a control: it is printed as the command is already being + prepared, and nobody reads a log line fast enough to stop it. So a recorded + config that carries shell is REFUSED unless the operator opted in. The common + case — ``execute`` then ``evaluate`` on your own machine — is unaffected + whenever the criteria are file/JSON checks, and the opt-in is one flag. + + Passing the task file explicitly (``evaluate ``) also + bypasses this: that config came from the operator, not from the artifact. """ - commands = [cmd for c in task.success_criteria if isinstance(cmd := getattr(c, "command", None), str)] - commands += [c.command for c in task.pre_run] + [c.command for c in task.post_run] + commands = embedded_commands(task, include_hooks=include_hooks) if not commands: return + rendered = "; ".join(commands) + if not allow_recorded_commands: + raise RegradeError( + f"The config recorded in {run_dir / TASK_JSON_FILENAME} would run {len(commands)} shell " + + f"command(s) on this host with your environment: {rendered}\n" + + "A run directory is a shareable artifact, so its recorded config is untrusted input. " + + "Re-run with --allow-recorded-commands to accept them, or pass the task file " + + "explicitly: coder-eval evaluate " + ) logger.warning( "Grading %s runs %d shell command(s) taken from that run's own recorded config: %s", run_dir, len(commands), - "; ".join(commands), + rendered, ) -def _fall_back_to_source(record: TaskConfigRecord, run_dir: Path, e: ValueError) -> tuple[TaskDefinition, str]: +def _fall_back_to_source( + record: TaskConfigRecord, + run_dir: Path, + e: ValueError, + *, + allow_recorded_commands: bool, + include_hooks: bool = True, +) -> tuple[TaskDefinition, str]: """The loud source-YAML fallback for a resolved config that no longer validates.""" from .task_loader import load_task @@ -120,7 +192,7 @@ def _fall_back_to_source(record: TaskConfigRecord, run_dir: Path, e: ValueError) record.source_file, ) task, source_yaml = load_task(Path(record.source_file)) - warn_on_embedded_commands(task, run_dir) + check_embedded_commands(task, run_dir, allow_recorded_commands=allow_recorded_commands, include_hooks=include_hooks) return task, source_yaml @@ -160,8 +232,20 @@ def default_workspace(run_dir: Path, prior: EvaluationResult) -> Path: # The exact path, not a heuristic. `task_id` may contain "/" (dataset rows # are "/"), so "the single child of artifacts/" resolves one # level too high for every row task. + # + # Containment-checked like the sandbox_path branch above, and for the same + # reason: `task_id` is an unvalidated string out of the run's own task.json, + # so `"../../../../home/victim"` joins to a real directory that `is_dir()` + # happily confirms. Every run_command criterion then executes with that as + # its cwd. The two branches read the same untrusted record; only one of them + # used to check. by_task_id = artifacts / prior.task_id if by_task_id.is_dir(): + if not _is_within(by_task_id, artifacts): + raise RegradeError( + f"The recorded task_id ({prior.task_id!r}) resolves outside {artifacts}. " + + "Pass --workspace explicitly to grade a directory outside the run." + ) return by_task_id children = [p for p in sorted(artifacts.iterdir()) if p.is_dir()] @@ -208,8 +292,6 @@ def verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition, ta + "Grading proceeds; a reference edited since the run would go undetected." ) return - from coder_eval.path_utils import digest_tree - from .evaluation import resolve_reference_dir try: @@ -224,7 +306,7 @@ def verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition, ta f"The reference directory recorded for this run is gone ({resolved}). Grading now " + "would score against a missing answer key. Restore it, or re-run the task." ) - if digest_tree(resolved) != recorded: + if _staged_digest(resolved) != recorded: raise RegradeError( f"The reference directory {resolved} changed since this run was executed " + "(digest mismatch). Grading now would score the agent's work against a " @@ -232,6 +314,30 @@ def verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition, ta ) +def _staged_digest(source: Path) -> str: + """Digest ``source`` the way the run recorded it — through a staged copy. + + The recorded ``reference_digest`` is taken over the per-run STAGED copy + (``Orchestrator._stage_reference``), which ``stage_reference_dir`` filters + through ``REFERENCE_COPY_IGNORE`` (``.git``) and strips of symlinks. + Digesting the raw source instead compares two differently-filtered trees, so + any reference that is a git checkout — the case the ignore list exists for — + reports a permanent false mismatch and un-grades the row for good. + + Re-staging rather than re-implementing the filter keeps the two in step: a + future entry in the ignore list applies here without a second edit. + """ + import tempfile + + from coder_eval.path_utils import digest_tree + + from .evaluation import stage_reference_dir + + with tempfile.TemporaryDirectory(prefix="coder-eval-refdigest-") as tmp: + staged = stage_reference_dir(source, Path(tmp) / "reference") + return digest_tree(staged) + + def back_up_pre_grade_record(run_dir: Path) -> None: """Keep the ungraded ``task.json`` beside the graded one, once. @@ -255,19 +361,90 @@ def back_up_pre_grade_record(run_dir: Path) -> None: logger.warning("Could not preserve the pre-grade record at %s: %s", backup, e) -def grading_sandbox_config(task: TaskDefinition) -> SandboxConfig: +def restore_pre_grade_record(run_dir: Path) -> bool: + """Put the ungraded ``task.json`` back after a grading crash. + + ``Orchestrator._finalize_result`` writes ``task.json`` into its run dir + *before* returning, so by the time a caller sees ``FinalStatus.ERROR`` the + ERROR row is already on disk whenever the grade wrote into the run being + graded (always, on ``run --resume``). Both commands treat ERROR as complete, + so the row would be permanently un-regradeable — and a caller that only fixes + its in-memory result leaves ``run.json`` disagreeing with ``task.json``. + + Returns whether the restore happened; there is nothing to restore when the + grade wrote elsewhere and the original was never replaced. + """ + source, backup = run_dir / TASK_JSON_FILENAME, run_dir / PRE_GRADE_JSON_FILENAME + if not backup.is_file() or backup.is_symlink() or source.is_symlink(): + return False + try: + text = backup.read_text(encoding="utf-8") + if source.is_file() and source.read_text(encoding="utf-8") == text: + return False # never overwritten; nothing to undo + write_text_atomic(source, text) + except OSError as e: + logger.warning("Could not restore the ungraded record at %s: %s", source, e) + return False + logger.info("Restored the ungraded record at %s after a grading failure.", source) + return True + + +def grading_sandbox_config(task: TaskDefinition, *, allow_host_grading: bool = False) -> SandboxConfig: """The sandbox config a grading pass runs under. Grading never runs a container: the docker driver dispatches through - DockerRunner, which needs an agent. Forcing ``tempdir`` keeps a task whose - YAML says ``driver: docker`` gradeable on the host. + DockerRunner, which needs an agent. So a ``driver: docker`` task can only be + graded on the host — and that is a DIFFERENT machine from the one its + criteria were written against. + + It is therefore refused rather than downgraded. A container task's criteria + address container paths (``/verifier``, ``/logs/verifier``) and container + toolchains; run on the host they score 0.0 for a trajectory ``run`` scored + 1.0, and the row is written back FAILURE. The same commands (``rm -rf + /verifier``, ``mkdir -p /logs/verifier``) also execute unsandboxed on the + grading machine. A silent rewrite additionally neutralized the ``docker`` + refusal in ``Sandbox.adopt``, which exists to catch exactly this. + + ``allow_host_grading`` is the operator's explicit acceptance of both. Rows + graded that way are stamped ``graded_on_host`` in ``environment_info`` + (:func:`stamp_host_grading`) so they are never silently comparable with rows + a container graded. Re-validated rather than ``model_copy(update=...)``: ``update`` skips both pydantic validation and pyright, so a typo would produce a SandboxConfig violating its own ``Literal`` and surface much later at an unrelated ``if driver == "docker"`` branch. """ - return SandboxConfig.model_validate({**task.sandbox.model_dump(), "driver": "tempdir"}) + if task.sandbox.driver != "docker": + return task.sandbox.model_copy(deep=True) + if not allow_host_grading: + raise RegradeError( + f"Task {task.task_id!r} ran under `driver: docker`, and grading cannot start a container " + + "(there is no agent to run in it). Grading on the host would execute this task's " + + "criteria against a filesystem that lacks the container's paths and toolchain, " + + "scoring a FAILURE for a run that passed — and would run its shell commands " + + "unsandboxed here. Re-run with --allow-host-grading to accept that, or grade on a " + + "machine that reproduces the container." + ) + logger.warning( + "Grading %r on the host: its `driver: docker` sandbox cannot be reproduced here, so " + + "path- and toolchain-dependent criteria may score differently than they did in the run. " + + "The row is stamped graded_on_host.", + task.task_id, + ) + return SandboxConfig.model_validate({**task.sandbox.model_dump(), "driver": "tempdir"}) # noqa: CE051 + + +def stamp_host_grading(result: EvaluationResult, task: TaskDefinition) -> None: + """Record that a docker task's verdict was produced on the host. + + Written onto the result, not just logged: a console warning does not travel + with ``task.json`` into ``run.json``, the reports or the evalboard, and this + row must never be compared with a container-graded one without that caveat + attached. + """ + if task.sandbox.driver == "docker": + result.environment_info["graded_on_host"] = True async def regrade_in_place( @@ -280,6 +457,7 @@ async def regrade_in_place( source_yaml: str, variant_id: str, replicate_index: int = 0, + allow_host_grading: bool = False, ) -> EvaluationResult: """Run ``task``'s criteria against an already-executed ``workspace``. @@ -300,7 +478,7 @@ async def regrade_in_place( verify_reference_unchanged(prior, task, task_file) sandbox = Sandbox( - grading_sandbox_config(task), + grading_sandbox_config(task, allow_host_grading=allow_host_grading), task_id=task.task_id, task_dir=task_file.parent.resolve() if task_file is not None else None, ) @@ -318,4 +496,6 @@ async def regrade_in_place( replicate_index=replicate_index, prior_result=prior, ) - return await orchestrator.run() + result = await orchestrator.run() + stamp_host_grading(result, task) + return result diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 306f3155..33420fe3 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -303,7 +303,6 @@ def build_task_event(result: EvaluationResult, *, driver: str, variant_id: str) "Status": result.final_status.value, "Category": result.final_status.category, "DurationMs": int((result.duration_seconds or 0.0) * 1000), - "Score": float(result.weighted_score or 0.0), "Iterations": result.iteration_count, "AgentType": result.agent_type or "", "Model": result.model_used or "", @@ -311,6 +310,13 @@ def build_task_event(result: EvaluationResult, *, driver: str, variant_id: str) "EarlyStopped": result.early_stop is not None, "EarlyStopReason": (result.early_stop.reason.value if result.early_stop is not None else ""), } + # `Score` is OMITTED, never coalesced to 0.0, when the row was not graded. + # Dashboards compute `avg(todouble(customDimensions.Score))` with no status + # filter, so a laundered zero for a `coder-eval execute` night drags every + # score tile toward zero and is indistinguishable from a genuinely bad night. + # An absent dimension drops out of the average instead. + if result.weighted_score is not None: + props["Score"] = float(result.weighted_score) return "CoderEval.Task.End", props @@ -516,6 +522,44 @@ def _agent_name(self) -> str: return str(self.task.agent.type) return AgentKind.NONE.value + def _terminal_status(self, success: bool) -> FinalStatus: + """The status a normally-completed evaluation loop lands on. + + Extracted from ``run()`` because the chain answers one question and + ``run()`` answers several; inlining it grew ``run()`` past the + complexity bound the moment the grading switch was threaded in. + + Order matters at every step: + + * A detached grade may NOT overturn an execution fact. The prior run's + terminal status (TIMEOUT, ERROR, a budget stop) describes an agent + phase this pass neither repeated nor observed. Without the first arm a + crashed run re-graded against its half-finished workspace reports + SUCCESS — with the original ``error_message`` still attached. + * The NOT_GRADED arm sits between the execution facts and FAILURE: under + ``grade=False`` no criterion ran, so ``success`` is always False and + FAILURE would be a verdict never actually reached — but + MAX_TURNS_EXHAUSTED is a fact about the RUN, like the statuses the + ``except`` branches assign, and still applies. + + With ``grade=True`` and no prior result the chain is the original one. + """ + assert self.result is not None, "Result not initialized" + inherited = self.prior_result.final_status if self.prior_result is not None else None + if inherited is not None and inherited.is_execution_fact: + logger.info( + "Preserving the run's terminal status %s: grading cannot overturn an execution fact.", + inherited.value, + ) + return inherited + if success: + return FinalStatus.SUCCESS + if self.result.max_turns_exhausted: + return FinalStatus.MAX_TURNS_EXHAUSTED + if not self.grade: + return FinalStatus.NOT_GRADED + return FinalStatus.FAILURE + async def run(self) -> EvaluationResult: """Run the complete evaluation. @@ -608,35 +652,7 @@ def _kill_agent_subprocess_sync() -> None: elapsed_seconds=time.time() - start_time, ) - # Update final status. The NOT_GRADED arm sits between the - # execution facts and FAILURE deliberately: under grade=False no - # criterion ran, so `success` is always False and FAILURE would be - # a verdict we never actually reached — but MAX_TURNS_EXHAUSTED - # (like TIMEOUT / BUILD_FAILED / the budget stops on the except - # branches below) is a fact about the RUN, not about grading, and - # still applies. With grade=True the chain is unchanged. - # - # A detached grade goes further: the prior run's terminal status - # may itself be an execution fact (TIMEOUT, ERROR, a budget stop) - # that this pass neither repeated nor observed, so grading must - # not overwrite it. Without this, a crashed run re-graded against - # its half-finished workspace reports SUCCESS — with the original - # error_message still attached. - inherited = self.prior_result.final_status if self.prior_result is not None else None - if inherited is not None and inherited.is_execution_fact: - logger.info( - "Preserving the run's terminal status %s: grading cannot overturn an execution fact.", - inherited.value, - ) - self.result.final_status = inherited - elif success: - self.result.final_status = FinalStatus.SUCCESS - elif self.result.max_turns_exhausted: - self.result.final_status = FinalStatus.MAX_TURNS_EXHAUSTED - elif not self.grade: - self.result.final_status = FinalStatus.NOT_GRADED - else: - self.result.final_status = FinalStatus.FAILURE + self.result.final_status = self._terminal_status(success) except asyncio.CancelledError: # Re-raise cancellation to allow proper task cancellation @@ -1103,10 +1119,13 @@ def _finalize_result(self, start_time: float) -> None: # Terminal per-task summary line. Emitted before report writes so a # write failure cannot swallow the one-line outcome. logger.info( - "Task finished: status=%s duration=%.1fs score=%.3f iterations=%d", + "Task finished: status=%s duration=%.1fs score=%s iterations=%d", self.result.final_status.value, self.result.duration_seconds or 0.0, - self.result.weighted_score or 0.0, + # "n/a", not 0.000: this line is read when diagnosing a run, and a + # zero here would say the criteria scored nothing rather than that + # nothing was scored. + "n/a" if self.result.weighted_score is None else f"{self.result.weighted_score:.3f}", self.result.iteration_count, ) @@ -1741,6 +1760,25 @@ def _reject_litellm_agent_judge_if_unsupported(self) -> None: ) raise ValueError(msg) + def _record_grader_route_provenance(self) -> None: + """Record the GRADING host's route without touching the run's. + + Two keys only — the agent route and the judge route — because those are + the two that decide what a re-grade's LLM criteria actually talked to. + Omitted entirely when they match the run, so an ordinary same-machine + re-grade adds nothing to the record. + """ + assert self.result is not None + assert self.route is not None + env = self.result.environment_info + grader_routes = {"graded_by_api_routing": ROUTE_NAMES[type(self.route)]} + if self.eval_route is not None: + grader_routes["graded_by_eval_routing"] = ROUTE_NAMES[type(self.eval_route)] + for key, value in grader_routes.items(): + run_key = key.removeprefix("graded_by_") + if env.get(run_key) != value: + env[key] = value + def _record_route_environment_info(self) -> None: """Persist resolved route + judge transport into ``result.environment_info``. @@ -1750,6 +1788,17 @@ def _record_route_environment_info(self) -> None: """ assert self.result is not None assert self.route is not None + if self.prior_result is not None: + # A detached grade resolves routes for ITS OWN host, which may be a + # different backend from the one that ran the task. Writing them into + # the run's keys contradicts the "prior wins" contract in + # _seed_from_prior_result and leaves a self-contradictory record — + # `api_routing: anthropic_direct` beside the run's stale `aws_region` + # and `bedrock_model`. Keep the run's routing; record the grader's + # alongside it, under the same `graded_by_` provenance prefix the + # seeding uses, and only when it actually differs. + self._record_grader_route_provenance() + return self.result.environment_info["api_routing"] = ROUTE_NAMES[type(self.route)] # The judge side (llm_judge / agent_judge) may run on a different, # constant backend — pinned to Claude when the agent is on LiteLLM — so @@ -2076,25 +2125,46 @@ def _sanitize_restored_path(self, recorded: str) -> str: runs in. Prepending it verbatim lets a run dir decide which binary ``pytest`` resolves to on the grader's host. - Two filters, both cheap and both about what PATH parity actually needs: - drop anything that is not an existing directory (a dead entry buys no - parity), and drop any entry inside the workspace being graded (that tree is - agent-writable, so a shim dropped there would shadow a real tool). What - remains is the run's genuine toolchain locations. + Four filters, all cheap and all about what PATH parity actually needs: + + * **Absolute only.** A relative entry resolves against the grader's + *current working directory*, which has nothing to do with the run — so + ``evilbin`` in a recorded PATH becomes ``$PWD/evilbin`` at the front of + every criterion subprocess's PATH. It also cannot be the toolchain + location it claims to be, since the run resolved it somewhere else. + * Drop anything that is not an existing directory (a dead entry buys no + parity). + * Drop any entry inside the **workspace** being graded — that tree is + agent-writable, so a shim dropped there would shadow a real tool. + * Drop any entry inside the **run directory** as a whole. The workspace + is only part of it; ``artifacts/``, a sibling replicate's tree and the + run root itself all travel in the same shared artifact and are all + equally attacker-chosen. + + What remains is the run's genuine toolchain locations. """ workspace = self.sandbox.sandbox_dir.resolve() if self.sandbox and self.sandbox.sandbox_dir else None + run_root = self.run_dir.resolve() + blocked = [p for p in (workspace, run_root) if p is not None] kept: list[str] = [] for entry in recorded.split(os.pathsep): if not entry: continue candidate = Path(entry) + if not candidate.is_absolute(): + logger.warning( + "Dropping recorded PATH entry %r: it is relative, so it would resolve against " + + "the grader's working directory rather than the run's toolchain.", + entry, + ) + continue if not candidate.is_dir(): logger.debug("Dropping recorded PATH entry %s: not a directory here.", entry) continue resolved = candidate.resolve() - if workspace is not None and (resolved == workspace or workspace in resolved.parents): + if any(resolved == root or root in resolved.parents for root in blocked): logger.warning( - "Dropping recorded PATH entry %s: it lies inside the workspace being graded, " + "Dropping recorded PATH entry %s: it lies inside the run being graded, " + "so a binary there could shadow a real tool on the grader's host.", entry, ) @@ -2247,6 +2317,21 @@ async def _evaluation_loop(self) -> bool: logger.debug(f"Agent response received ({len(turn_record.agent_output)} chars)") + # Facts about the RUN, recorded before the grading switch. `execute` + # withholds the verdict, never the facts: a max-turns or over-budget run + # must finalize the same way under `execute` as under `run`, or + # `execute` exits 0 where `run` exits 1 for identical agent output — and + # `_seed_from_prior_result` cannot restore a fact the execute phase never + # captured, so a later `evaluate` inherits the wrong terminal status too. + if turn_record.max_turns_exhausted: + self.result.max_turns_exhausted = True + logger.warning( + "Agent exhausted max_turns (%s).", + self.task.run_limits.max_turns if self.task.run_limits else None, + ) + # Soft cumulative-turn check (logs once; never aborts). + self._check_expected_turns(iteration=iteration) + # Grading site 2 of 4. `execute` stops here: the trajectory is captured # and persisted exactly as on a graded run, but nothing is scored. # Returning False keeps FinalStatus off SUCCESS; run()'s status chain @@ -2254,6 +2339,10 @@ async def _evaluation_loop(self) -> bool: # — it exists to protect a grade that is not happening. if not self.grade: logger.info("Grading disabled (execute mode): skipping success criteria.") + # The budget gate is a run limit, not a verdict. Its only reason to + # sit after the criteria on the graded path is partial-credit + # visibility, and there is no partial credit here. + self._check_run_limits(iteration=iteration) return False # Check success criteria (reference_dir feeds reference_comparison + judges) @@ -2276,23 +2365,18 @@ async def _evaluation_loop(self) -> bool: # Reuse the model method for weighted score (single source of truth) self.result.calculate_weighted_score(self.task.success_criteria) - current_score = self.result.weighted_score or 0.0 + # calculate_weighted_score just ran, so a score exists; the fallback is + # for the type, not for an unmeasured row (this branch only runs when + # grading did). + current_score = self.result.weighted_score or 0.0 # noqa: CE049 — graded here by construction logger.info(f"Success criteria: {passed_count}/{total_count} passed, weighted score: {current_score:.3f}") self._emit_criteria_event(criteria_results) - if turn_record.max_turns_exhausted: - self.result.max_turns_exhausted = True - logger.warning( - "Agent exhausted max_turns (%s) without passing criteria.", - self.task.run_limits.max_turns if self.task.run_limits else None, - ) - - # Soft cumulative-turn check (logs once; never aborts). - self._check_expected_turns(iteration=iteration) - # Budget gate runs AFTER criteria so partial-credit visibility is preserved. + # (max_turns capture and the soft turn check are recorded above, before + # the grading switch — they are facts about the run, not verdicts.) self._check_run_limits(iteration=iteration) return all_passed @@ -2780,7 +2864,8 @@ def _emit_criteria_event(self, criteria_results: list[CriterionResult]) -> None: pairs = list(zip(criteria_results, self.task.success_criteria, strict=True)) passed_count = sum(1 for r, c in pairs if r.score >= c.pass_threshold) total_count = len(pairs) - current_score = self.result.weighted_score or 0.0 + # Reached only from the grading path, where a score has been computed. + current_score = self.result.weighted_score or 0.0 # noqa: CE049 — graded here by construction criteria_details = [ f"{criterion.type}: {'PASS' if result.score >= criterion.pass_threshold else 'FAIL'}" + f" ({result.score:.2f})" diff --git a/src/coder_eval/path_utils.py b/src/coder_eval/path_utils.py index c0a3b39f..ba99620d 100644 --- a/src/coder_eval/path_utils.py +++ b/src/coder_eval/path_utils.py @@ -46,10 +46,32 @@ def write_text_atomic(path: Path, text: str) -> None: agent again, and the row vanishes from ``run.json``. One writer, so the orchestrator and the detached grade's write-back cannot have different crash semantics for the same file. + + The temp file is opened ``O_CREAT | O_EXCL | O_NOFOLLOW``. Without it a + pre-planted ``task.json.tmp`` *symlink* in a shared run directory makes this + an arbitrary-file-overwrite primitive — and one that bypasses the destination + symlink refusal in ``evaluate``'s write-back, since the guard checks the + destination while the truncation happens through the temp name. A partial + temp file is unlinked before the error propagates, so a failed write never + leaves ``.tmp`` litter beside the record. """ tmp = path.with_suffix(path.suffix + ".tmp") - tmp.write_text(text, encoding="utf-8") - os.replace(tmp, path) + flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0) + try: + fd = os.open(tmp, flags, 0o600) + except FileExistsError as e: + raise OSError( + f"Refusing to write {path}: {tmp} already exists. Remove it if it is stale — a " + + "pre-planted temp file (especially a symlink) would redirect this write." + ) from e + try: + with os.fdopen(fd, "w", encoding="utf-8") as fh: + fh.write(text) + os.replace(tmp, path) + except BaseException: + with contextlib.suppress(OSError): + os.unlink(tmp) + raise def digest_tree(root: Path) -> str: diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py index c734f3d0..4762ea65 100644 --- a/src/coder_eval/reports.py +++ b/src/coder_eval/reports.py @@ -913,8 +913,10 @@ def _compute_suite_rollup( per_rows = [ row.result.success_criteria_results[i] for row in rows if i < len(row.result.success_criteria_results) ] - suite_thresholds = getattr(criterion, "suite_thresholds", None) - description = getattr(criterion, "description", None) + # Both are declared on BaseSuccessCriterion, so they are typed + # attributes on every union member — no string probe needed. + suite_thresholds = criterion.suite_thresholds + description = criterion.description try: checker_cls = CriterionRegistry.get_checker(ctype) except KeyError: @@ -942,7 +944,12 @@ def _compute_suite_rollup( # Sample up to K failed/errored rows for error analysis failed_samples: list[FailedRowSummary] = [] for row in rows: - if row.result.final_status.category == "succeeded": + # "succeeded" is not the only non-failure. An UNGRADED row was never + # measured, so it has no failure reasons to report and listing it here + # (in a field documented as failed/errored rows) contradicts the same + # function's own rule two blocks up, where it leaves both sides of the + # pass rate. + if row.result.final_status.category in ("succeeded", "ungraded"): continue if len(failed_samples) >= _FAILED_SAMPLE_LIMIT: break @@ -987,7 +994,9 @@ def _compute_suite_rollup( rows_failed=rows_failed, rows_error=rows_error, rows_not_graded=rows_not_graded, - pass_rate=rows_passed / rows_graded if rows_graded else 0.0, + # None, never 0.0: a suite where nothing was graded has no pass rate, + # and 0.0 renders as "0.0%" beside a full set of rows. + pass_rate=rows_passed / rows_graded if rows_graded else None, average_weighted_score=average_weighted_score, criterion_stats=criterion_stats, failed_samples=failed_samples, @@ -1007,7 +1016,11 @@ def _render_suite_markdown(rollup: SuiteRollup) -> str: + f"{rollup.rows_failed} failed, {rollup.rows_error} errored" + (f", {rollup.rows_not_graded} not graded" if rollup.rows_not_graded else "") ), - f"**Pass rate**: {rollup.pass_rate * 100:.1f}%", + ( + f"**Pass rate**: {rollup.pass_rate * 100:.1f}% ({rollup.rows_passed}/{rollup.rows_graded})" + if rollup.pass_rate is not None + else "**Pass rate**: n/a (no rows were graded)" + ), ] if rollup.average_weighted_score is not None: lines.append(f"**Average weighted score**: {rollup.average_weighted_score:.3f}") @@ -1202,12 +1215,12 @@ def write_suite_rollups( (suite_dir / "suite.md").write_text(_render_suite_markdown(rollup), encoding="utf-8") rollups.append(rollup) logger.info( - "Wrote suite rollup: variant=%s suite=%s pass_rate=%.1f%% (%d/%d) gate=%s", + "Wrote suite rollup: variant=%s suite=%s pass_rate=%s (%d/%d) gate=%s", variant_id, suite_id, - rollup.pass_rate * 100, + f"{rollup.pass_rate * 100:.1f}%" if rollup.pass_rate is not None else "n/a", rollup.rows_passed, - rollup.rows_total, + rollup.rows_graded, "PASS" if rollup.passed else "FAIL", ) return rollups diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index a10fb5f7..dc3b5eb6 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -346,7 +346,19 @@ def _aggregate_count_rows(result: ExperimentResult, show_p_values: bool) -> list row += " | —" lines.append(row + " |") - # Every task the variant ran is in the denominator, errors included. + # Row: Not Graded — conditional, like the budget sub-rows above. Without + # it Tasks Run / Succeeded / Failed / Errors stop summing to tasks_run on + # an ungraded run, with nothing in the table to say where the rest went. + if any(result.variant_aggregates[vid].tasks_not_graded > 0 for vid in result.variant_ids): + row = "| Not Graded" + for vid in result.variant_ids: + row += f" | {result.variant_aggregates[vid].tasks_not_graded}" + if show_p_values: + row += " | —" + lines.append(row + " |") + + # Every task the variant GRADED is in the denominator, errors included; + # ungraded rows leave both sides. row = "| Pass Rate" for vid in result.variant_ids: rate = result.variant_aggregates[vid].pass_rate diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py index 39e7a600..7f8fcade 100644 --- a/src/coder_eval/reports_html.py +++ b/src/coder_eval/reports_html.py @@ -1359,6 +1359,17 @@ def _row(label: str, values: list[str], p: str | None) -> str: ) rows.append(_row("Failed", [str(result.variant_aggregates[vid].tasks_failed) for vid in result.variant_ids], None)) rows.append(_row("Errors", [str(result.variant_aggregates[vid].tasks_error) for vid in result.variant_ids], None)) + # The fourth bucket, conditionally like its siblings elsewhere. Without it + # Tasks Run / Succeeded / Failed / Errors no longer sum to tasks_run on an + # ungraded run, with nothing on the page to say where the rest went. + if any(result.variant_aggregates[vid].tasks_not_graded > 0 for vid in result.variant_ids): + rows.append( + _row( + "Not Graded", + [str(result.variant_aggregates[vid].tasks_not_graded) for vid in result.variant_ids], + None, + ) + ) def _pass_rate(vid: str) -> str: rate = result.variant_aggregates[vid].pass_rate diff --git a/src/coder_eval/reports_stats.py b/src/coder_eval/reports_stats.py index 78306b5c..18ee2751 100644 --- a/src/coder_eval/reports_stats.py +++ b/src/coder_eval/reports_stats.py @@ -348,15 +348,22 @@ def collect_variant_series(result: ExperimentResult) -> dict[str, VariantSeries] s = series.get(vr.variant_id) if s is None: # a task result for a variant not in variant_ids continue - if vr.weighted_score is None: - # Ungraded row: no score exists, and appending 0.0 would enter a - # fabricated data point into every statistic below. Skip the row - # WHOLE rather than just its score — paired_comparison pairs the - # series across variants by index, so dropping one field would - # misalign them. `grade` is run-level, so an experiment is either - # entirely graded or entirely ungraded; this never splits a pair. - continue - s.scores.append(vr.weighted_score) + # Only the SCORE is dropped when there is none — never the row. + # Duration, tokens and assistant turns are facts about the run that + # grading has nothing to do with, and `execute`'s stated contract is + # that only the verdict is withheld. Skipping the row whole made an + # all-ungraded experiment render `Avg Duration | N/A | N/A` with the + # Tokens and Assistant Turns rows absent entirely. + # + # The series are consumed independently (each statistic reads one + # list), so they need not be index-aligned with each other; + # `paired_comparison` pairs across VARIANTS by task id, not by index + # into these lists. An earlier note here claimed an experiment is + # either entirely graded or entirely ungraded because `grade` is + # run-level — `run --resume` grades rows independently and folds a + # failed one back ungraded, so mixed experiments are real. + if vr.weighted_score is not None: + s.scores.append(vr.weighted_score) s.durations.append(vr.duration_seconds / vr.replicate_count) if vr.total_tokens is not None: s.tokens.append(float(vr.total_tokens)) diff --git a/tests/lint/rules/ce047_env_info_key_round_trip.py b/tests/lint/rules/ce047_env_info_key_round_trip.py index 1afe6281..6638077a 100644 --- a/tests/lint/rules/ce047_env_info_key_round_trip.py +++ b/tests/lint/rules/ce047_env_info_key_round_trip.py @@ -59,7 +59,11 @@ def _written_keys() -> set[str]: class EnvInfoKeyRoundTrip(BaseRule): id = "CE047" - _SRC_PATH = re.compile(r"[/\\]src[/\\]coder_eval[/\\]") + # `(^|sep)`, not a bare leading separator: a repo-relative path + # ("src/coder_eval/x.py") is how every caller in tests/ addresses a file, + # and requiring the separator silently put those out of scope — which + # would make a rule-behaviour test pass while proving nothing. + _SRC_PATH = re.compile(r"(?:^|[/\\])src[/\\]coder_eval[/\\]") _written: set[str] | None = None def __init__(self, filepath: str) -> None: diff --git a/tests/lint/rules/ce049_no_score_or_zero.py b/tests/lint/rules/ce049_no_score_or_zero.py new file mode 100644 index 00000000..49153617 --- /dev/null +++ b/tests/lint/rules/ce049_no_score_or_zero.py @@ -0,0 +1,70 @@ +"""CE049: never coalesce a possibly-unmeasured score to a numeric literal. + +``weighted_score is None`` means *nothing measured this row*, and it is a +different fact from ``weighted_score == 0.0``, which means *this row was measured +and scored nothing*. ``score or 0.0`` erases that difference — and it does it +silently, producing a real-looking number that every downstream consumer treats +as a genuine miss. + +The motivating bug: ``build_task_event`` published ``Score = float( +result.weighted_score or 0.0)`` on every ``CoderEval.Task.End``. Four shipped +App Insights tiles compute ``avg(todouble(customDimensions.Score))`` with no +status filter, so one ``coder-eval execute`` night dragged every score tile +toward zero, indistinguishable from a genuinely bad night. The hazard was +already documented in prose in ``orchestrator.py`` ("every downstream +`score or 0.0` would launder it into a real-looking failure") — this rule makes +it mechanical. + +Fires on `` or `` where the left operand's trailing name +looks like a score or a rate. The fix is to omit the value, keep it ``None``, or +branch explicitly on ``is None``. + +``# noqa: CE049`` for a genuinely aggregate-internal use where a missing value +really is a miss — e.g. summing a variant's scores where an errored row must +count as 0.0 (see ``orchestration/experiment._measured_scores``, which makes that +decision explicitly and states why). +""" + +import ast +import re + +from tests.lint.rules.base import BaseRule + + +# Trailing-segment match, so `result.weighted_score`, `v.average_score` and +# `summary.pass_rate` all fire while `sample_rate_limit` does not. +_SCORE_NAME = re.compile(r"^(weighted_score|score|average_score|pass_rate|[a-z_]*_rate)$") + + +def _scoreish(node: ast.expr) -> str | None: + """The name of a score/rate-looking operand, or None.""" + if isinstance(node, ast.Attribute): + name = node.attr + elif isinstance(node, ast.Name): + name = node.id + else: + return None + return name if _SCORE_NAME.match(name) else None + + +def _numeric_literal(node: ast.expr) -> bool: + return isinstance(node, ast.Constant) and isinstance(node.value, int | float) and not isinstance(node.value, bool) + + +class NoScoreOrZero(BaseRule): + id = "CE049" + + def visit_BoolOp(self, node: ast.BoolOp) -> None: + if isinstance(node.op, ast.Or) and len(node.values) == 2: + name = _scoreish(node.values[0]) + if name is not None and _numeric_literal(node.values[1]): + self.violation( + node, + f"{name!r} is coalesced to a numeric literal. `None` means the row was never " + + "measured; a literal means it was measured and scored that value — and the " + + "coalesce publishes the second while meaning the first, which is how an " + + "ungraded run reached four avg(Score) dashboards as a real zero. Omit the " + + "value, keep None, or branch on `is None`. Use `# noqa: CE049` where a " + + "missing value genuinely IS a miss, with a comment saying so.", + ) + self.generic_visit(node) diff --git a/tests/lint/rules/ce050_no_union_getattr_probe.py b/tests/lint/rules/ce050_no_union_getattr_probe.py new file mode 100644 index 00000000..166d6be2 --- /dev/null +++ b/tests/lint/rules/ce050_no_union_getattr_probe.py @@ -0,0 +1,132 @@ +"""CE050: no untyped ``getattr`` probe for a field of a discriminated union. + +``getattr(criterion, "command", None)`` reads as "the members that have a +command". It is not: it is a string the type checker cannot see. Rename +``RunCommandCriterion.command`` and pyright reports nothing, ruff reports +nothing, and the probe silently returns ``None`` forever — the guard it powers +becomes a permanent no-op with every gate green. + +``models/tasks.py`` already states the rule in prose, verbatim: "isinstance +narrowing, NOT getattr(c, 'files'/'command'): with an untyped string probe, +renaming ... turns this load-time guard into a silent no-op that pyright cannot +see." This promotes that convention to a gate. + +The motivating bug: ``orchestration/regrade.warn_on_embedded_commands`` — the +only disclosure of what shell a rebuilt, untrusted run config would execute on +the grader's host — probed with ``getattr(c, "command", None)``. Besides being +rename-fragile it structurally could not name ``agent_judge``, the criterion +that spawns a tool-using agent and therefore has the widest blast radius of all. + +Fires on ``getattr(, "", ...)`` in ``src/coder_eval/`` where the +literal is a field name declared by a member of one of the tracked discriminated +unions AND ```` is named like a criterion / template source / route. The +field list is derived from the models at collection time, so it tracks renames +instead of going stale. + +The receiver-name filter is deliberate, and it is the rule's known limit. Field +names like ``command``, ``tool`` and ``prompt`` are far too common to flag on +their own — the agents legitimately probe raw SDK event objects for exactly those +— so a name-only rule would fire a dozen times on code that has nothing to do +with these unions and would be turned off within a week. Scoping to the +receiver's name catches the real shape (``for c in task.success_criteria: ... +getattr(c, "command", None)``) and leaves an unusual receiver name uncovered. + +The fix is ``isinstance`` narrowing. ``# noqa: CE050`` for a probe that really is +duck-typed across unrelated objects. +""" + +import ast +import re + +from tests.lint.rules.base import BaseRule + + +def _union_field_names() -> set[str]: + """Field names declared by any member of the tracked unions. + + Derived from the models rather than hardcoded: a hardcoded list is the same + class of staleness the rule exists to prevent. Names shared with ordinary + object attributes (``type``, ``description``, ``weight`` …) are excluded — + they are not what a rename would break, and flagging them would only teach + people to noqa the rule. + """ + import typing + + from coder_eval.models import ApiRoute, SuccessCriterion, TemplateSource + + common = {"type", "description", "weight", "pass_threshold", "model", "path"} + + def _members(annotation: object) -> list[object]: + """Flatten ``Annotated[Union[...], Field(discriminator=...)]`` to its members. + + The unions are all discriminated, so a bare ``__args__`` yields + ``(Union[...], FieldInfo)`` and the model classes are one level deeper — + which silently produced an EMPTY field set, i.e. a rule that could never + fire. Recurse instead. + """ + args = typing.get_args(annotation) + if not args: + return [annotation] + out: list[object] = [] + for arg in args: + out.extend(_members(arg) if typing.get_args(arg) else [arg]) + return out + + names: set[str] = set() + for union in (SuccessCriterion, TemplateSource, ApiRoute): + for member in _members(union): + names |= set(getattr(member, "model_fields", {})) + assert names, "CE050 derived no union field names — the rule could never fire" + return names - common + + +# Receivers that name a member of one of the tracked unions. See the module +# docstring for why the rule is scoped this way rather than on the field alone. +_RECEIVER = re.compile(r"^(c|cr|crit|criterion|source|template_source|route|api_route)$|(_criterion|_source|_route)$") + + +def _union_receiver(node: ast.expr) -> bool: + """Whether the probed object is named like a union member.""" + if isinstance(node, ast.Name): + return bool(_RECEIVER.search(node.id)) + if isinstance(node, ast.Attribute): + return bool(_RECEIVER.search(node.attr)) + return False + + +class NoUnionGetattrProbe(BaseRule): + id = "CE050" + + # `(^|sep)` so a repo-relative path is in scope too; see CE047. + _SRC_PATH = re.compile(r"(?:^|[/\\])src[/\\]coder_eval[/\\]") + _fields: set[str] | None = None + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = bool(self._SRC_PATH.search(filepath)) + if self._in_scope and NoUnionGetattrProbe._fields is None: + NoUnionGetattrProbe._fields = _union_field_names() + + def visit_Call(self, node: ast.Call) -> None: + self._check(node) + self.generic_visit(node) + + def _check(self, node: ast.Call) -> None: + if not self._in_scope: + return + if not isinstance(node.func, ast.Name) or node.func.id != "getattr" or len(node.args) < 2: + return + key = node.args[1] + if not isinstance(key, ast.Constant) or not isinstance(key.value, str): + return + if key.value not in (NoUnionGetattrProbe._fields or set()): + return + if not _union_receiver(node.args[0]): + return + self.violation( + node, + f"getattr(..., {key.value!r}) probes a discriminated-union field with an untyped " + + "string. pyright cannot see it, so a rename turns this into a silent no-op that " + + "returns None forever — and it cannot reach members that express the same " + + "capability under a different field. Narrow with isinstance instead.", + ) diff --git a/tests/lint/rules/ce051_no_driver_override.py b/tests/lint/rules/ce051_no_driver_override.py new file mode 100644 index 00000000..f6943139 --- /dev/null +++ b/tests/lint/rules/ce051_no_driver_override.py @@ -0,0 +1,94 @@ +"""CE051: a sandbox driver may not be rewritten silently. + +The driver IS the isolation boundary. Rewriting ``docker`` to ``tempdir`` behind +the caller's back does not degrade gracefully — it moves execution from a +container onto the operator's own machine, where the task's criteria address +paths and toolchains that do not exist. They score 0.0 and the row is written +back FAILURE for a trajectory that passed, and the same commands (``rm -rf +/verifier``, ``mkdir -p /logs/verifier``) run unsandboxed on the grading host. + +The motivating bug: ``regrade.grading_sandbox_config`` rewrote the driver +unconditionally on BOTH new grading entry points, which also neutralized the +``driver: docker`` refusal in ``Sandbox.adopt`` — a guard added in the same +change specifically to catch this. + +A driver downgrade must be an explicit, logged, operator-visible decision. Fires +on any construction that carries an existing sandbox config forward while +replacing ``driver``: + + * ``SandboxConfig.model_validate({**cfg.model_dump(), "driver": ...})`` + * ``cfg.model_copy(update={"driver": ...})`` + * ``setattr(cfg, "driver", ...)`` / ``cfg.driver = ...`` + +Exempt: ``models/sandbox.py`` (the model's own construction), and any site +carrying ``# noqa: CE051`` with a reason — today the two legitimate ones are the +in-container rewrite in ``run_task_internal_command`` and the opt-in host-grading +branch, which refuses by default and stamps ``graded_on_host`` on the row. +""" + +import ast +import re + +from tests.lint.rules.base import BaseRule + + +_MESSAGE = ( + "This rewrites `driver` on an existing sandbox config. The driver is the isolation " + "boundary: silently moving a docker task onto the host makes its criteria address paths " + "and a toolchain that are not there, so they score 0.0 and the row is written back FAILURE " + "for a run that passed — and its shell runs unsandboxed on this machine. Refuse, or make it " + "an explicit opt-in that stamps the row, and add `# noqa: CE051` naming the reason." +) + + +def _has_driver_key(node: ast.expr) -> bool: + """True when ``node`` is a dict/dict-display whose keys include ``"driver"``.""" + if not isinstance(node, ast.Dict): + return False + return any(isinstance(k, ast.Constant) and k.value == "driver" for k in node.keys if k is not None) + + +class NoDriverOverride(BaseRule): + id = "CE051" + + # The model's own module legitimately constructs and defaults the field. + _EXEMPT_PATH = re.compile(r"[/\\]models[/\\]sandbox\.py$") + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = bool( + re.search(r"(?:^|[/\\])src[/\\]coder_eval[/\\]", filepath) + ) and not self._EXEMPT_PATH.search(filepath) + + def visit_Call(self, node: ast.Call) -> None: + if self._in_scope: + self._check_call(node) + self.generic_visit(node) + + def visit_Assign(self, node: ast.Assign) -> None: + if self._in_scope: + for target in node.targets: + if isinstance(target, ast.Attribute) and target.attr == "driver": + self.violation(node, _MESSAGE) + self.generic_visit(node) + + def _check_call(self, node: ast.Call) -> None: + # cfg.model_copy(update={"driver": ...}) + if isinstance(node.func, ast.Attribute) and node.func.attr == "model_copy": + for kw in node.keywords: + if kw.arg == "update" and _has_driver_key(kw.value): + self.violation(node, _MESSAGE) + return + # SandboxConfig.model_validate({**cfg.model_dump(), "driver": ...}) + if isinstance(node.func, ast.Attribute) and node.func.attr == "model_validate" and node.args: + arg = node.args[0] + # Only a SPREAD dict — a literal built from scratch is an ordinary + # construction, not a rewrite of somebody else's config. + if _has_driver_key(arg) and isinstance(arg, ast.Dict) and any(k is None for k in arg.keys): + self.violation(node, _MESSAGE) + return + # setattr(cfg, "driver", ...) + if isinstance(node.func, ast.Name) and node.func.id == "setattr" and len(node.args) >= 2: + key = node.args[1] + if isinstance(key, ast.Constant) and key.value == "driver": + self.violation(node, _MESSAGE) diff --git a/tests/lint/runner.py b/tests/lint/runner.py index 383c3064..f302da08 100644 --- a/tests/lint/runner.py +++ b/tests/lint/runner.py @@ -28,6 +28,9 @@ from tests.lint.rules.ce046_env_info_spreads_super import EnvInfoSpreadsSuper from tests.lint.rules.ce047_env_info_key_round_trip import EnvInfoKeyRoundTrip from tests.lint.rules.ce048_no_in_process_typer_command_call import NoInProcessTyperCommandCall +from tests.lint.rules.ce049_no_score_or_zero import NoScoreOrZero +from tests.lint.rules.ce050_no_union_getattr_probe import NoUnionGetattrProbe +from tests.lint.rules.ce051_no_driver_override import NoDriverOverride from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore @@ -79,6 +82,9 @@ EnvInfoSpreadsSuper, EnvInfoKeyRoundTrip, NoInProcessTyperCommandCall, + NoScoreOrZero, + NoUnionGetattrProbe, + NoDriverOverride, ] # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 8329d629..9583e945 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -3608,3 +3608,159 @@ def test_yaml_walk_flags_a_literal_tail_behind_a_variable(self, tmp_path: Path): encoding="utf-8", ) assert not self._offending_paths_in(task) + + +class TestCE047EnvInfoKeyRoundTrip: + """CE047 fires when an environment_info key is read with no writer anywhere. + + The rule shipped with only the whole-tree "finds nothing" scan, which cannot + tell a rule that is CORRECT from one that can never fire — the exact failure + its own docstring is about. The path form matters too: the rule scopes itself + with a leading-separator regex, so a repo-relative path must still be in + scope or a house-style test would pass vacuously. + """ + + @staticmethod + def _run(src: str, filepath: str = "src/coder_eval/orchestrator.py"): + import ast + + from tests.lint.rules.ce047_env_info_key_round_trip import EnvInfoKeyRoundTrip + + return EnvInfoKeyRoundTrip(filepath).check(ast.parse(src)) + + def test_flags_a_get_with_no_writer(self): + assert self._run('x = self.result.environment_info.get("no_such_key_anywhere")') + + def test_flags_a_subscript_read_with_no_writer(self): + assert self._run('x = self.result.environment_info["no_such_key_anywhere"]') + + def test_allows_a_key_that_is_written_in_src(self): + # reference_digest gained a writer; that is the whole point of the rule. + assert not self._run('x = self.result.environment_info.get("reference_digest")') + + def test_allows_the_graded_by_prefix(self): + assert not self._run('x = self.result.environment_info.get("graded_by_git_commit")') + + def test_ignores_a_computed_key(self): + assert not self._run("x = self.result.environment_info.get(key)") + + def test_is_out_of_scope_outside_src(self): + assert not self._run('x = r.environment_info.get("no_such_key_anywhere")', filepath="tests/test_thing.py") + + def test_scope_is_the_same_for_relative_and_absolute_paths(self): + """A repo-relative path must be in scope, or every house-style test lies.""" + src = 'x = self.result.environment_info.get("no_such_key_anywhere")' + relative = self._run(src, filepath="src/coder_eval/orchestrator.py") + absolute = self._run(src, filepath="/home/u/repo/src/coder_eval/orchestrator.py") + assert bool(relative) == bool(absolute) is True + + +class TestCE048NoInProcessTyperCommandCall: + """CE048 fires on an in-process call to a Typer command function.""" + + @staticmethod + def _run(src: str, filepath: str = "tests/test_thing.py"): + import ast + + from tests.lint.rules.ce048_no_in_process_typer_command_call import NoInProcessTyperCommandCall + + return NoInProcessTyperCommandCall(filepath).check(ast.parse(src)) + + def test_flags_calling_a_command_function_directly(self): + assert self._run("from coder_eval.cli.evaluate_command import evaluate_command\nevaluate_command(x)") + + def test_allows_the_plain_python_entry_point(self): + assert not self._run("from coder_eval.cli.evaluate_command import run_evaluation\nrun_evaluation(x=1)") + + +class TestCE049NoScoreOrZero: + """CE049 flags coalescing an unmeasured score into a real-looking number.""" + + @staticmethod + def _run(src: str): + import ast + + from tests.lint.rules.ce049_no_score_or_zero import NoScoreOrZero + + return NoScoreOrZero("src/coder_eval/orchestrator.py").check(ast.parse(src)) + + def test_flags_weighted_score_or_zero(self): + assert self._run("x = float(result.weighted_score or 0.0)") + + def test_flags_a_bare_score_name_and_an_int_literal(self): + assert self._run("x = score or 0") + + def test_flags_a_rate(self): + assert self._run("x = summary.pass_rate or 0.0") + + def test_allows_an_explicit_none_branch(self): + assert not self._run("x = 0.0 if result.weighted_score is None else result.weighted_score") + + def test_allows_a_non_numeric_fallback(self): + assert not self._run('x = result.weighted_score or "n/a"') + + def test_ignores_an_unrelated_name(self): + assert not self._run("x = retry_count or 0") + + +class TestCE050NoUnionGetattrProbe: + """CE050 flags an untyped getattr probe for a discriminated-union field.""" + + @staticmethod + def _run(src: str, filepath: str = "src/coder_eval/orchestration/regrade.py"): + import ast + + from tests.lint.rules.ce050_no_union_getattr_probe import NoUnionGetattrProbe + + return NoUnionGetattrProbe(filepath).check(ast.parse(src)) + + def test_flags_the_command_probe(self): + assert self._run('cmd = getattr(c, "command", None)') + + def test_allows_isinstance_narrowing(self): + assert not self._run("cmd = c.command if isinstance(c, RunCommandCriterion) else None") + + def test_ignores_a_name_no_union_member_declares(self): + assert not self._run('x = getattr(obj, "definitely_not_a_criterion_field", None)') + + def test_ignores_a_computed_key(self): + assert not self._run("x = getattr(obj, name, None)") + + def test_is_out_of_scope_outside_src(self): + assert not self._run('cmd = getattr(c, "command", None)', filepath="tests/test_thing.py") + + +class TestCE051NoDriverOverride: + """CE051 flags a silent sandbox-driver rewrite.""" + + @staticmethod + def _run(src: str, filepath: str = "src/coder_eval/orchestration/regrade.py"): + import ast + + from tests.lint.rules.ce051_no_driver_override import NoDriverOverride + + return NoDriverOverride(filepath).check(ast.parse(src)) + + def test_flags_a_spread_model_validate(self): + assert self._run('SandboxConfig.model_validate({**task.sandbox.model_dump(), "driver": "tempdir"})') + + def test_flags_model_copy_update(self): + assert self._run('cfg.model_copy(update={"driver": "tempdir"})') + + def test_flags_setattr(self): + assert self._run('setattr(cfg, "driver", "tempdir")') + + def test_flags_attribute_assignment(self): + assert self._run('cfg.driver = "tempdir"') + + def test_allows_a_config_built_from_scratch(self): + assert not self._run('SandboxConfig.model_validate({"driver": "tempdir"})') + + def test_allows_carrying_a_config_forward_unchanged(self): + assert not self._run("task.sandbox.model_copy(deep=True)") + + def test_is_out_of_scope_in_the_model_module(self): + assert not self._run( + 'cfg.model_copy(update={"driver": "tempdir"})', + filepath="src/coder_eval/models/sandbox.py", + ) diff --git a/tests/test_detached_grading_boundaries.py b/tests/test_detached_grading_boundaries.py new file mode 100644 index 00000000..278297d1 --- /dev/null +++ b/tests/test_detached_grading_boundaries.py @@ -0,0 +1,423 @@ +"""The guards that decide WHERE and WHETHER a detached grade runs. + +Three boundaries, each shipped without a behavioural test: + +* the docker ``grade`` boundary — the only thing standing between + ``execute --driver docker`` against a stale image and a run that silently + publishes real verdicts; +* ``grading_sandbox_config`` — which decides whether a container task's criteria + may run on the grading host at all; +* the crash-recovery arm — the one a REAL grading failure takes, which is not the + one the existing test exercises (see ``TestGradingCrashLeavesTheRowRegradeable``). +""" + +from __future__ import annotations + +import json +from datetime import datetime +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from coder_eval.cli import app +from coder_eval.models import ( + AgentKind, + EvaluationResult, + FileExistsCriterion, + FinalStatus, + ResolvedTask, + SandboxConfig, + TaskDefinition, + parse_agent_config, +) +from coder_eval.orchestration.regrade import ( + RegradeError, + grading_sandbox_config, + restore_pre_grade_record, + stamp_host_grading, +) +from coder_eval.path_utils import PRE_GRADE_JSON_FILENAME, TASK_JSON_FILENAME + + +runner = CliRunner() + + +def _task(driver: str = "tempdir") -> TaskDefinition: + return TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + sandbox=SandboxConfig(driver=driver), # type: ignore[arg-type] + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + + +def _result(status: FinalStatus = FinalStatus.NOT_GRADED) -> EvaluationResult: + return EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2026, 1, 1), + final_status=status, + iteration_count=0, + ) + + +# -------------------------------------------------------------------------- +# grading_sandbox_config: a docker task may not be graded on the host by default +# -------------------------------------------------------------------------- + + +class TestGradingSandboxConfig: + def test_a_docker_task_is_refused_by_default(self) -> None: + """Grading cannot start a container, and grading on the host runs the + task's criteria against a filesystem without the container's paths or + toolchain — scoring FAILURE for a trajectory `run` scored 1.0, and + executing its shell unsandboxed here.""" + with pytest.raises(RegradeError) as exc: + grading_sandbox_config(_task("docker")) + assert "--allow-host-grading" in str(exc.value) + + def test_the_opt_in_downgrades_to_tempdir(self) -> None: + cfg = grading_sandbox_config(_task("docker"), allow_host_grading=True) + assert cfg.driver == "tempdir" + + def test_a_non_docker_task_is_carried_through_unchanged(self) -> None: + """No rewrite at all on the ordinary path — the config the run used IS + the config the grade uses.""" + task = _task("tempdir") + cfg = grading_sandbox_config(task) + assert cfg.driver == "tempdir" + assert cfg == task.sandbox + + def test_a_host_graded_docker_row_is_stamped(self) -> None: + """A console warning does not travel with task.json into run.json, the + reports or the evalboard. The row must carry the caveat itself, or it is + silently comparable with a container-graded one.""" + result = _result() + stamp_host_grading(result, _task("docker")) + assert result.environment_info["graded_on_host"] is True + + def test_a_normal_row_carries_no_stamp(self) -> None: + result = _result() + stamp_host_grading(result, _task("tempdir")) + assert "graded_on_host" not in result.environment_info + + +# -------------------------------------------------------------------------- +# The docker `grade` boundary +# -------------------------------------------------------------------------- + + +def _docker_runner(*, grade: bool, tmp_path: Path): + from coder_eval.isolation.docker_runner import DockerRunner + + rt = ResolvedTask( + task=_task("docker"), + task_file=tmp_path / "t.yaml", + run_dir=tmp_path / "run", + variant_id="default", + original_task_id="t", + ) + return DockerRunner(rt, grade=grade) + + +class TestDockerGradeBoundary: + """`grade` crosses the container boundary only through context.json, so an + image that predates `execute` ignores the key and grades anyway.""" + + def test_a_graded_verdict_from_an_execute_run_is_refused(self, tmp_path: Path) -> None: + from coder_eval.isolation.docker_runner import DockerRunError + + runner_ = _docker_runner(grade=False, tmp_path=tmp_path) + with pytest.raises(DockerRunError, match="predates `execute`"): + runner_._assert_grade_honored(_result(FinalStatus.SUCCESS)) + + def test_an_ungraded_row_is_accepted(self, tmp_path: Path) -> None: + _docker_runner(grade=False, tmp_path=tmp_path)._assert_grade_honored(_result()) + + def test_an_execution_fact_is_exempt(self, tmp_path: Path) -> None: + """TIMEOUT / ERROR describe the agent phase, not grading. `execute` + reports them exactly as `run` does, so they are not evidence the image + graded anything.""" + for status in (FinalStatus.TIMEOUT, FinalStatus.ERROR, FinalStatus.BUILD_FAILED): + _docker_runner(grade=False, tmp_path=tmp_path)._assert_grade_honored(_result(status)) + + def test_a_graded_run_short_circuits(self, tmp_path: Path) -> None: + _docker_runner(grade=True, tmp_path=tmp_path)._assert_grade_honored(_result(FinalStatus.SUCCESS)) + + +class TestInContainerGradeCoercion: + """The container side of the same boundary.""" + + @staticmethod + def _run_with_context(tmp_path: Path, grade: object): + input_dir = tmp_path / "input" + input_dir.mkdir() + # Only the keys read BEFORE the grade coercion need real values; the + # command must refuse before it ever builds an Orchestrator. + context = {"variant_id": "default", "source_yaml": "task_id: t\n", "grade": grade} + (input_dir / "context.json").write_text(json.dumps(context), encoding="utf-8") + (input_dir / "task.yaml").write_text("task_id: t\n", encoding="utf-8") + return runner.invoke( + app, + ["_run-task-internal", "--input", str(input_dir), "--output", str(tmp_path / "out")], + ) + + def test_a_non_boolean_grade_is_a_hard_error(self, tmp_path: Path) -> None: + """A hand-edited or older-format `"grade": "false"` is a truthy str typed + as bool, which would silently grade a run that asked not to be.""" + result = self._run_with_context(tmp_path, "false") + assert result.exit_code == 2 + assert "must be a boolean" in result.output + + def test_the_default_is_to_grade(self) -> None: + """A host predating `execute` writes no key; the container must keep its + original behaviour rather than silently withholding verdicts.""" + import inspect + + from coder_eval.cli.run_task_internal_command import run_task_internal_command + + source = inspect.getsource(inspect.getmodule(run_task_internal_command)) # type: ignore[arg-type] + assert 'context.get("grade", True)' in source + + +# -------------------------------------------------------------------------- +# The crash-recovery arm +# -------------------------------------------------------------------------- + + +class TestGradingCrashLeavesTheRowRegradeable: + """``Orchestrator.run()`` converts an internal failure into a populated + ``FinalStatus.ERROR`` result rather than raising, so a REAL grading crash + takes the ``else:`` arm — not the ``except`` arm the pre-existing test + exercises by patching ``regrade_in_place`` to raise. + + Both arms must leave ``task.json`` ungraded. ERROR is "complete" for both + commands, so an ERROR row written over a NOT_GRADED one can never be graded + again without hand-restoring ``task.execute.json``. + """ + + @staticmethod + def _run_dir_with_backup(tmp_path: Path) -> Path: + run_dir = tmp_path / "00" + run_dir.mkdir(parents=True) + ungraded = _result().model_dump_json(indent=2) + (run_dir / PRE_GRADE_JSON_FILENAME).write_text(ungraded, encoding="utf-8") + # What _finalize_result already wrote before the caller saw the ERROR. + (run_dir / TASK_JSON_FILENAME).write_text( + _result(FinalStatus.ERROR).model_dump_json(indent=2), encoding="utf-8" + ) + return run_dir + + def test_restore_puts_the_ungraded_record_back(self, tmp_path: Path) -> None: + run_dir = self._run_dir_with_backup(tmp_path) + assert restore_pre_grade_record(run_dir) is True + on_disk = EvaluationResult.model_validate_json((run_dir / TASK_JSON_FILENAME).read_text(encoding="utf-8")) + assert on_disk.final_status is FinalStatus.NOT_GRADED + + def test_restore_is_a_no_op_when_nothing_was_overwritten(self, tmp_path: Path) -> None: + """The grade wrote into a fresh run dir, so the original is untouched.""" + run_dir = tmp_path / "00" + run_dir.mkdir(parents=True) + text = _result().model_dump_json(indent=2) + (run_dir / PRE_GRADE_JSON_FILENAME).write_text(text, encoding="utf-8") + (run_dir / TASK_JSON_FILENAME).write_text(text, encoding="utf-8") + assert restore_pre_grade_record(run_dir) is False + + def test_restore_refuses_to_write_through_a_symlink(self, tmp_path: Path) -> None: + run_dir = self._run_dir_with_backup(tmp_path) + victim = tmp_path / "victim.json" + victim.write_text("keep me", encoding="utf-8") + (run_dir / TASK_JSON_FILENAME).unlink() + (run_dir / TASK_JSON_FILENAME).symlink_to(victim) + + assert restore_pre_grade_record(run_dir) is False + assert victim.read_text(encoding="utf-8") == "keep me" + + async def test_the_resume_error_arm_restores_and_folds_back_ungraded(self, tmp_path: Path) -> None: + """The arm a real grading crash takes: `regrade_in_place` RETURNS an + ERROR result instead of raising.""" + from coder_eval.cli.run_command import _grade_resumed_tasks + + # task.json starts UNGRADED, as `execute` left it. The ERROR lands on + # disk during the grade, exactly as _finalize_result writes it before + # returning — which is the whole reason the in-memory fix is not enough. + run_dir = tmp_path / "00" + run_dir.mkdir(parents=True) + (run_dir / TASK_JSON_FILENAME).write_text(_result().model_dump_json(indent=2), encoding="utf-8") + rt = ResolvedTask( + task=_task(), + task_file=tmp_path / "t.yaml", + run_dir=run_dir, + variant_id="v", + original_task_id="t", + ) + + async def _crash(**_kwargs) -> EvaluationResult: + errored = _result(FinalStatus.ERROR) + (run_dir / TASK_JSON_FILENAME).write_text(errored.model_dump_json(indent=2), encoding="utf-8") + return errored + + with ( + patch("coder_eval.orchestration.regrade.default_workspace", return_value=tmp_path), + patch("coder_eval.orchestration.regrade.regrade_in_place", new=_crash), + ): + graded = await _grade_resumed_tasks([rt]) + + assert len(graded) == 1 + folded = graded[0][1].result + assert folded.final_status is FinalStatus.NOT_GRADED + assert "Grading errored during --resume" in (folded.error_message or "") + # And the on-disk row too — fixing only the in-memory result leaves + # run.json disagreeing with task.json, and task.json is what a later + # --resume reads. + on_disk = EvaluationResult.model_validate_json((run_dir / TASK_JSON_FILENAME).read_text(encoding="utf-8")) + assert on_disk.final_status is FinalStatus.NOT_GRADED + + async def test_an_unreadable_row_still_appears_as_ungraded(self, tmp_path: Path) -> None: + """Dropping it removed it from run.json AND from `tasks_not_graded`, + which is the counter the exit gate reads — so a resume whose rows were + all unreadable reported success.""" + from coder_eval.cli.run_command import _grade_resumed_tasks + + run_dir = tmp_path / "00" + run_dir.mkdir(parents=True) + (run_dir / TASK_JSON_FILENAME).write_text("{not json", encoding="utf-8") + rt = ResolvedTask( + task=_task(), + task_file=tmp_path / "t.yaml", + run_dir=run_dir, + variant_id="v", + original_task_id="t", + ) + + graded = await _grade_resumed_tasks([rt]) + + assert len(graded) == 1 + assert graded[0][1].result.final_status is FinalStatus.NOT_GRADED + assert "could not be read" in (graded[0][1].result.error_message or "") + + +# -------------------------------------------------------------------------- +# write_text_atomic hardening +# -------------------------------------------------------------------------- + + +class TestAtomicWriteIsNotAnOverwritePrimitive: + """``evaluate``'s write-back guards its DESTINATION against a symlink, but the + truncation happens through the temp name — so a pre-planted + ``task.json.tmp`` symlink bypassed the guard entirely.""" + + def test_a_pre_planted_temp_symlink_is_refused(self, tmp_path: Path) -> None: + from coder_eval.path_utils import write_text_atomic + + victim = tmp_path / "victim" + victim.write_text("keep me", encoding="utf-8") + target = tmp_path / "task.json" + (tmp_path / "task.json.tmp").symlink_to(victim) + + with pytest.raises(OSError, match="already exists"): + write_text_atomic(target, "attacker content") + assert victim.read_text(encoding="utf-8") == "keep me" + + def test_an_ordinary_write_still_works(self, tmp_path: Path) -> None: + from coder_eval.path_utils import write_text_atomic + + target = tmp_path / "task.json" + write_text_atomic(target, "hello") + assert target.read_text(encoding="utf-8") == "hello" + assert not (tmp_path / "task.json.tmp").exists() + + def test_a_failed_write_leaves_no_temp_file(self, tmp_path: Path) -> None: + from coder_eval.path_utils import write_text_atomic + + target = tmp_path / "task.json" + with patch("os.replace", side_effect=OSError("boom")), pytest.raises(OSError): + write_text_atomic(target, "hello") + assert not (tmp_path / "task.json.tmp").exists() + + +# -------------------------------------------------------------------------- +# default_workspace traversal +# -------------------------------------------------------------------------- + + +def test_a_task_id_that_escapes_the_artifacts_tree_is_refused(tmp_path: Path) -> None: + """`task_id` is an unvalidated string out of the run's own task.json, and + `artifacts / "../../.."` joins to a real directory that `is_dir()` confirms. + Every run_command criterion would then execute with that as its cwd. The + sibling `sandbox_path` branch was containment-checked; this one was not.""" + from coder_eval.orchestration.regrade import default_workspace + + (tmp_path / "run" / "artifacts").mkdir(parents=True) + (tmp_path / "outside").mkdir() + + prior = _result() + prior.task_id = "../../outside" + + with pytest.raises(RegradeError, match="resolves outside"): + default_workspace(tmp_path / "run", prior) + + +# -------------------------------------------------------------------------- +# The reference digest +# -------------------------------------------------------------------------- + + +def test_the_reference_digest_is_computed_over_a_staged_copy(tmp_path: Path) -> None: + """The recorded digest is taken over the per-run STAGED copy, which strips + `.git`. Digesting the raw source instead compares two differently-filtered + trees, so any reference that is a git checkout — the case the ignore list + exists for — reports a permanent false mismatch and un-grades the row. + """ + from coder_eval.orchestration.evaluation import stage_reference_dir + from coder_eval.orchestration.regrade import _staged_digest + from coder_eval.path_utils import digest_tree + + source = tmp_path / "reference" + (source / ".git").mkdir(parents=True) + (source / ".git" / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8") + (source / "solution.py").write_text("print('hi')\n", encoding="utf-8") + + staged = stage_reference_dir(source, tmp_path / "staged") + recorded = digest_tree(staged) # exactly what Orchestrator._stage_reference records + + assert _staged_digest(source) == recorded + # And the naive comparison this replaced would have failed. + assert digest_tree(source) != recorded + + +def test_a_reference_edited_since_the_run_is_still_caught(tmp_path: Path) -> None: + """The guard must not become permissive: only the FILTER changed, not what + counts as a change.""" + from coder_eval.orchestration.regrade import _staged_digest + + source = tmp_path / "reference" + source.mkdir() + (source / "solution.py").write_text("print('hi')\n", encoding="utf-8") + before = _staged_digest(source) + + (source / "solution.py").write_text("print('tampered')\n", encoding="utf-8") + assert _staged_digest(source) != before + + +def test_a_reference_that_vanished_is_refused_not_skipped(tmp_path: Path) -> None: + """A missing answer key must raise, not return silently — grading against + nothing produces an ordinary-looking score.""" + from coder_eval.orchestration.regrade import verify_reference_unchanged + + task = _task() + task.reference = MagicMock() + prior = _result() + prior.environment_info["reference_digest"] = "deadbeef" + + with ( + patch("coder_eval.orchestration.evaluation.resolve_reference_dir", return_value=tmp_path / "gone"), + pytest.raises(RegradeError, match="is gone"), + ): + verify_reference_unchanged(prior, task, tmp_path / "t.yaml") diff --git a/tests/test_detached_grading_guards.py b/tests/test_detached_grading_guards.py index d0f66b18..6f3c75b4 100644 --- a/tests/test_detached_grading_guards.py +++ b/tests/test_detached_grading_guards.py @@ -202,14 +202,23 @@ def test_the_agents_path_is_persisted_so_a_later_grade_can_restore_it(tmp_path: assert "command_base_path" in orch.result.environment_info -def test_a_restored_path_drops_entries_inside_the_graded_workspace(tmp_path: Path) -> None: +def test_a_restored_path_drops_entries_inside_the_graded_run(tmp_path: Path) -> None: """The restored value is PREPENDED ahead of the host PATH and comes out of the - run's own task.json. An entry inside the agent-writable workspace could - shadow a real tool on the grader's host.""" - workspace = tmp_path / "ws" + run's own task.json — a shareable artifact. Every entry an attacker could + have placed there must be dropped; only the run's real toolchain survives. + + The run-directory SIBLING case is the one this test used to pin the wrong way + round: it asserted such an entry was kept. The workspace is only part of the + run dir, and ``artifacts/`` and the run root travel in the same archive. + """ + run_dir = tmp_path / "run" + workspace = run_dir / "ws" (workspace / "bin").mkdir(parents=True) - outside = tmp_path / "toolchain" - outside.mkdir() + sibling = run_dir / "artifacts-shim" # inside the run dir, outside the workspace + sibling.mkdir() + toolchain = tmp_path / "toolchain" # a genuine location outside the run entirely + toolchain.mkdir() + relative = Path("evilbin") task = TaskDefinition( task_id="t", @@ -218,18 +227,28 @@ def test_a_restored_path_drops_entries_inside_the_graded_workspace(tmp_path: Pat agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), success_criteria=[FileExistsCriterion(path="x.txt", description="x")], ) - orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v") + orch = Orchestrator(task=task, run_dir=run_dir, variant_id="v") orch.sandbox = MagicMock() orch.sandbox.sandbox_dir = workspace # os.pathsep, not a hardcoded ":" — the separator is ";" on Windows, where a # colon-joined value parses as one (non-existent) entry and every assertion # below passes vacuously against an empty result. - recorded = os.pathsep.join([str(workspace / "bin"), str(outside), str(tmp_path / "gone")]) + recorded = os.pathsep.join( + [ + str(workspace / "bin"), + str(sibling), + str(relative), + str(toolchain), + str(tmp_path / "gone"), + ] + ) kept = orch._sanitize_restored_path(recorded) - assert str(outside.resolve()) in kept - assert str(workspace) not in kept, "an entry inside the graded tree must be dropped" + assert str(toolchain.resolve()) in kept, "a real out-of-run toolchain entry is the point of the restore" + assert str(workspace) not in kept, "an entry inside the graded workspace must be dropped" + assert str(sibling) not in kept, "an entry elsewhere in the run directory must be dropped too" + assert "evilbin" not in kept, "a relative entry would resolve against the grader's cwd" assert "gone" not in kept, "a non-existent entry buys no parity" diff --git a/tests/test_execute_command.py b/tests/test_execute_command.py index e7686c6b..ae124104 100644 --- a/tests/test_execute_command.py +++ b/tests/test_execute_command.py @@ -224,6 +224,7 @@ def _option_names(command: str) -> set[str]: # or consciously listed below as a deliberate omission. _DELIBERATELY_ABSENT_FROM_EXECUTE = { "--junit-xml", # a report of verdicts, and there are none + "--allow-host-grading", # decides how a GRADE runs; execute never grades } diff --git a/tests/test_execute_evaluate_loop.py b/tests/test_execute_evaluate_loop.py index cd6f566d..bdd62c1f 100644 --- a/tests/test_execute_evaluate_loop.py +++ b/tests/test_execute_evaluate_loop.py @@ -358,3 +358,71 @@ def test_grading_the_same_run_twice_reaches_the_same_verdict(tmp_path: Path) -> assert second["sandbox_path"] == first["sandbox_path"], "the artifacts pointer must survive a re-grade" # The pre-grade record is still the ORIGINAL ungraded one, not the first grade's. assert _row(task_dir, "task.execute.json")["final_status"] == FinalStatus.NOT_GRADED.value + + +# -------------------------------------------------------------------------- +# `execute` withholds the verdict, never the facts of the run +# -------------------------------------------------------------------------- + + +def test_execute_records_max_turns_exhausted_exactly_as_run_does(tmp_path: Path) -> None: + """`max_turns_exhausted` is a fact about the RUN, not a verdict. + + It used to be captured AFTER the grading switch's early return, so under + `execute` it was never recorded at all: the row finalized NOT_GRADED and the + command exited 0 where `run` reported MAX_TURNS_EXHAUSTED and exited 1 — for + identical agent output. `_seed_from_prior_result` cannot restore a fact the + execute phase never captured, so a later `evaluate` inherited the wrong + terminal status too. + """ + from coder_eval.streaming.collector import EventCollector + + # One turn that reports the cap was hit, on both paths. + original = EventCollector.build_turn_record + + def _exhausted(self, *args: Any, **kwargs: Any): + record = original(self, *args, **kwargs) + record.max_turns_exhausted = True + return record + + def _run(command: str, run_dir: Path) -> Any: + with patch.object(EventCollector, "build_turn_record", _exhausted): + return runner.invoke(app, [command, str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + + graded_dir = tmp_path / "graded" + _run("run", graded_dir) + graded = _row(_task_dir(graded_dir)) + + executed_dir = tmp_path / "executed" + _run("execute", executed_dir) + executed = _row(_task_dir(executed_dir)) + + assert graded["max_turns_exhausted"] is True, "the fixture must actually exhaust turns under `run`" + assert executed["max_turns_exhausted"] is True, ( + "`execute` dropped a fact about the run. Only the verdict is withheld." + ) + assert executed["final_status"] == FinalStatus.MAX_TURNS_EXHAUSTED.value + + +def test_a_detached_grade_keeps_the_runs_api_routing_not_the_graders(tmp_path: Path) -> None: + """`_seed_from_prior_result`'s contract is that the PRIOR run wins on + environment_info. The route recorder ran after the seeding and overwrote + `api_routing` with the grading host's, leaving a self-contradictory record — + a direct route named beside the run's stale bedrock fields.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + + before = _row(task_dir)["environment_info"] + before["api_routing"] = "a_backend_this_host_does_not_use" + row = _row(task_dir) + row["environment_info"] = before + (task_dir / "task.json").write_text(json.dumps(row), encoding="utf-8") + + _invoke(["evaluate", str(task_dir)]) + after = _row(task_dir)["environment_info"] + + assert after["api_routing"] == "a_backend_this_host_does_not_use", ( + "the grade overwrote the RUN's recorded routing with the grading host's" + ) + assert after.get("graded_by_api_routing"), "the grader's own route must still be recorded, just not in place" diff --git a/tests/test_experiment_runner.py b/tests/test_experiment_runner.py index 1ee6f08d..a9259fa5 100644 --- a/tests/test_experiment_runner.py +++ b/tests/test_experiment_runner.py @@ -750,3 +750,84 @@ def test_all_replicates_errored_gives_zero_duration(self): vr = result.task_summaries[0].variant_results[0] assert abs(vr.duration_seconds - 0.0) < 1e-9 assert vr.final_status == FinalStatus.ERROR + + +class TestAggregationCountsErroredRowsAsMisses: + """An ERRORED row must count as 0.0, not vanish from both sides. + + ``weighted_score is None`` and ``final_status.category == "ungraded"`` look + interchangeable and are not: an ERROR / BUILD_FAILED row also has no score. + Filtering on the score dropped it from the numerator AND the denominator, so + a nightly where one image build failed reported a HIGHER headline score than + a clean one, and A/B comparisons were biased toward whichever variant errored + more. Only an ungraded row leaves both sides — nothing measured it. + """ + + @staticmethod + def _result(task_id: str, variant_id: str, status: str, score: float | None) -> EvaluationResult: + return EvaluationResult( + task_id=task_id, + task_description="d", + variant_id=variant_id, + agent_type="claude-code", + started_at=datetime.now(), + final_status=status, # type: ignore[arg-type] + weighted_score=score, + duration_seconds=1.0, + iteration_count=1, + environment_info={}, + ) + + def _aggregate(self, rows: list[tuple[str, str, str, float | None]]) -> ExperimentResult: + return aggregate_results( + experiment_id="e", + description="d", + variant_ids=sorted({vid for _, vid, _, _ in rows}), + task_results=[ + TaskResult( + task_id=task_id, + variant_id=variant_id, + result=self._result(task_id, variant_id, status, score), + duration=1.0, + ) + for task_id, variant_id, status, score in rows + ], + total_duration=1.0, + ) + + def test_an_errored_row_drags_the_average_down_like_a_miss(self): + clean = self._aggregate([("a", "v", "SUCCESS", 1.0), ("b", "v", "FAILURE", 0.0)]) + errored = self._aggregate([("a", "v", "SUCCESS", 1.0), ("b", "v", "ERROR", None)]) + + assert clean.variant_aggregates["v"].average_score == 0.5 + assert errored.variant_aggregates["v"].average_score == 0.5, ( + "an infrastructure-failure night must not score HIGHER than a clean one" + ) + + def test_a_build_failure_is_a_miss_too(self): + result = self._aggregate([("a", "v", "SUCCESS", 1.0), ("b", "v", "BUILD_FAILED", None)]) + assert result.variant_aggregates["v"].average_score == 0.5 + + def test_an_ungraded_row_leaves_both_sides(self): + result = self._aggregate([("a", "v", "SUCCESS", 1.0), ("b", "v", "NOT_GRADED", None)]) + assert result.variant_aggregates["v"].average_score == 1.0 + + def test_a_fully_ungraded_variant_has_no_average(self): + result = self._aggregate([("a", "v", "NOT_GRADED", None)]) + assert result.variant_aggregates["v"].average_score is None + + def test_score_spread_still_sees_a_fully_errored_arm(self): + """A task where variant B fully errored reported spread 0.0 instead of + the real gap, because B had no score to compare.""" + result = self._aggregate([("a", "x", "SUCCESS", 0.8), ("a", "y", "ERROR", None)]) + assert result.task_summaries[0].score_spread == pytest.approx(0.8) + + def test_no_arbitrary_winner_when_nothing_was_scored(self): + """`variants[0]` named whichever arm the input happened to list first, + with `is_tie=False` asserting it was a real result — so swapping the + inputs flipped the reported winner.""" + one = self._aggregate([("a", "x", "NOT_GRADED", None), ("a", "y", "NOT_GRADED", None)]) + other = self._aggregate([("a", "y", "NOT_GRADED", None), ("a", "x", "NOT_GRADED", None)]) + + assert one.task_summaries[0].best_variant == other.task_summaries[0].best_variant + assert one.task_summaries[0].is_tie is True diff --git a/tests/test_regrade.py b/tests/test_regrade.py index 6663089b..be8da625 100644 --- a/tests/test_regrade.py +++ b/tests/test_regrade.py @@ -132,18 +132,45 @@ def test_source_fallback_is_loud(tmp_path: Path, caplog: pytest.LogCaptureFixtur assert "NOT reapplied" in caplog.text -def test_shell_commands_from_a_run_dir_config_are_announced(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: +def test_shell_commands_from_a_run_dir_config_are_refused_by_default(tmp_path: Path) -> None: """A run dir is a shareable artifact, and rebuilding from it decides what the - grader executes. Intended, but never silent.""" + grader executes with the grader's credentials. + + A warning is not a control — it is printed as the command is already being + prepared. So the default is REFUSAL, and the message names both the commands + and the way to accept them.""" resolved = _task(command="echo surprising").model_dump(mode="json") prior = _result(task_config=TaskConfigRecord(resolved=resolved, source_yaml="raw", source_file=None)) - with caplog.at_level(logging.WARNING): + with pytest.raises(RegradeError) as exc: task_from_prior(prior, tmp_path) + assert "echo surprising" in str(exc.value) + assert "--allow-recorded-commands" in str(exc.value) + + +def test_the_opt_in_accepts_them_and_still_names_them(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + """Accepted is not the same as invisible: the opt-in still logs what will run.""" + resolved = _task(command="echo surprising").model_dump(mode="json") + prior = _result(task_config=TaskConfigRecord(resolved=resolved, source_yaml="raw", source_file=None)) + + with caplog.at_level(logging.WARNING): + task, _ = task_from_prior(prior, tmp_path, allow_recorded_commands=True) + + assert task.task_id assert "echo surprising" in caplog.text +def test_a_config_with_no_shell_needs_no_opt_in(tmp_path: Path) -> None: + """The common case — execute then evaluate your own file/JSON criteria — is + unaffected, or the gate would just be turned off.""" + resolved = _task().model_dump(mode="json") + prior = _result(task_config=TaskConfigRecord(resolved=resolved, source_yaml="raw", source_file=None)) + + task, _ = task_from_prior(prior, tmp_path) + assert task.task_id + + # -------------------------------------------------------------------------- # default_workspace # -------------------------------------------------------------------------- diff --git a/tests/test_reports_html.py b/tests/test_reports_html.py index 17c77255..b468d8a0 100644 --- a/tests/test_reports_html.py +++ b/tests/test_reports_html.py @@ -1261,3 +1261,79 @@ def test_status_badge_unknown_string_is_neutral(): badge = _status_badge("LEGACY_UNKNOWN") assert 'class="badge neutral"' in badge assert "LEGACY_UNKNOWN" in badge + + +class TestUngradedRenderingInHtml: + """The four ungraded changes to reports_html, only one of which had a test. + + An all-ungraded ExperimentResult exercises the variant tile, the variant + table's denominator, `_variant_stddev_lines`' None filter and the per-task + comparison's switch to `format_score` in one pass. + """ + + @staticmethod + def _all_ungraded(variant_ids: list[str]) -> ExperimentResult: + summaries = [ + TaskExperimentSummary( + task_id="t0", + variant_results=[ + VariantResult( + variant_id=vid, + task_id="t0", + weighted_score=None, + final_status=FinalStatus.NOT_GRADED, + duration_seconds=1.0, + total_tokens=100, + iteration_count=1, + total_assistant_turns=1, + ) + for vid in variant_ids + ], + best_variant=variant_ids[0], + is_tie=True, + score_spread=0.0, + ) + ] + aggregates = { + vid: VariantAggregate( + variant_id=vid, + tasks_run=3, + tasks_succeeded=0, + tasks_failed=0, + tasks_error=0, + tasks_not_graded=3, + average_score=None, + average_duration=12.0, + ) + for vid in variant_ids + } + return ExperimentResult( + experiment_id="exp-ungraded", + description="d", + variant_ids=variant_ids, + task_summaries=summaries, + variant_aggregates=aggregates, + total_duration_seconds=30.0, + ) + + def test_the_variant_tile_names_the_fourth_bucket(self): + """Rendered only when non-zero, so an off-by-one on `> 0` is invisible on + a graded run — this is the only place it shows.""" + html = HTMLReportGenerator.generate_experiment_html(self._all_ungraded(["v1"]), None) + assert "Not Graded" in html + + def test_a_graded_run_does_not_grow_the_tile(self): + html = HTMLReportGenerator.generate_experiment_html(_experiment_result(["v1"]), None) + assert "Not Graded" not in html + + def test_the_aggregate_table_accounts_for_every_task(self): + """Tasks Run / Succeeded / Failed / Errors stopped summing to tasks_run + on an ungraded run, with nothing on the page to say where the rest went.""" + html = HTMLReportGenerator.generate_experiment_html(self._all_ungraded(["v1", "v2"]), None) + assert "Not Graded" in html + + def test_no_surface_publishes_a_fabricated_zero(self): + """A run nothing measured must read `n/a`, never `0.0` or `0.0%`.""" + html = HTMLReportGenerator.generate_experiment_html(self._all_ungraded(["v1"]), None) + assert "n/a" in html + assert "0.0%" not in html diff --git a/tests/test_route_seam_exhaustiveness.py b/tests/test_route_seam_exhaustiveness.py index 3d04d762..422ee8ea 100644 --- a/tests/test_route_seam_exhaustiveness.py +++ b/tests/test_route_seam_exhaustiveness.py @@ -92,8 +92,16 @@ def test_record_route_environment_info_handles_every_route(): pyright can't check the isinstance chain. Also covers the simulator_route -> simulator_routing seam directly (not just via a shared alias with route).""" for r in _INSTANCES: + # prior_result=None: a detached grade takes a different branch that + # records the GRADING host's route under graded_by_* instead of + # overwriting the run's. This test covers the ordinary run. fake = SimpleNamespace( - route=r, eval_route=r, simulator_route=r, result=SimpleNamespace(environment_info={}), agent=None + route=r, + eval_route=r, + simulator_route=r, + result=SimpleNamespace(environment_info={}), + agent=None, + prior_result=None, ) Orchestrator._record_route_environment_info(fake) # type: ignore[arg-type] env = fake.result.environment_info diff --git a/tests/test_suite_rollup.py b/tests/test_suite_rollup.py index 425ee7d7..7adb9c7b 100644 --- a/tests/test_suite_rollup.py +++ b/tests/test_suite_rollup.py @@ -6,6 +6,8 @@ from datetime import datetime from pathlib import Path +import pytest + from coder_eval.models import ( AgentKind, ClassificationCriterionResult, @@ -210,7 +212,11 @@ def test_error_count_per_criterion(self, tmp_path: Path) -> None: def test_empty_suite(self, tmp_path: Path) -> None: rollup = _compute_suite_rollup("s", "v1", [], tmp_path) assert rollup.rows_total == 0 - assert rollup.pass_rate == 0.0 + # None, not 0.0. A suite that measured nothing has no pass rate, and + # 0.0 publishes "0.0%" — indistinguishable from a suite where every row + # failed. Same rule as RunSummary.pass_rate and VariantAggregate. + assert rollup.pass_rate is None + assert rollup.rows_graded == 0 assert rollup.average_weighted_score is None @@ -746,3 +752,74 @@ def test_missing_aggregator_threshold_check_matches_injected_metric(self, tmp_pa assert cr_check.actual_value == agg.metrics["completion_rate"] assert cr_check.passed is False assert rollup.passed is False + + +def _row(row_id: str, final_status: FinalStatus, weighted_score: float | None) -> TaskResult: + """A one-criterion suite row. An ungraded row carries NO criteria results — + that is what `execute` leaves behind, and grading them is the whole point.""" + criteria = [] if weighted_score is None else [("file_exists", weighted_score, None)] + return _make_row( + suite_id="s", + row_id=row_id, + final_status=final_status, + weighted_score=weighted_score, + criteria=criteria, + ) + + +class TestSuiteRollupUngradedBucket: + """The fourth bucket on the suite surface, which shipped with no coverage. + + An ungraded row (`coder-eval execute`) leaves BOTH sides of the suite pass + rate — it is not a pass and not a failure — and it carries no failure reason, + so it must not appear in `failed_samples` either. The row-count invariant + counts it, so a row landing outside all four buckets fails loudly instead of + silently vanishing from the rollup. + """ + + def test_ungraded_rows_leave_both_sides_of_the_pass_rate(self, tmp_path: Path) -> None: + rows = [ + _row("r1", FinalStatus.SUCCESS, 1.0), + _row("r2", FinalStatus.NOT_GRADED, None), + _row("r3", FinalStatus.NOT_GRADED, None), + ] + rollup = _compute_suite_rollup("s", "v1", rows, tmp_path) + + assert rollup.rows_total == 3 + assert rollup.rows_not_graded == 2 + assert rollup.rows_graded == 1 + # 1 of 1 graded, not 1 of 3. + assert rollup.pass_rate == 1.0 + + def test_a_fully_ungraded_suite_has_no_pass_rate(self, tmp_path: Path) -> None: + rows = [_row(f"r{i}", FinalStatus.NOT_GRADED, None) for i in range(3)] + rollup = _compute_suite_rollup("s", "v1", rows, tmp_path) + + assert rollup.rows_graded == 0 + assert rollup.pass_rate is None + assert "n/a" in _render_suite_markdown(rollup) + + def test_ungraded_rows_are_not_collected_as_failed_samples(self, tmp_path: Path) -> None: + """`failed_samples` is documented as failed/errored rows. An ungraded row + has no failure reasons to show, so listing it contradicts the same + function's pass-rate rule two blocks up.""" + rows = [ + _row("r1", FinalStatus.FAILURE, 0.0), + _row("r2", FinalStatus.NOT_GRADED, None), + ] + rollup = _compute_suite_rollup("s", "v1", rows, tmp_path) + + assert [s.row_id for s in rollup.failed_samples] == ["r1"] + + def test_a_row_outside_every_bucket_fails_the_invariant(self) -> None: + with pytest.raises(ValueError, match="Suite row count invariant violated"): + SuiteRollup( + suite_id="s", + variant_id="v", + rows_total=3, + rows_passed=1, + rows_failed=0, + rows_error=0, + rows_not_graded=1, # 1+0+0+1 != 3 + pass_rate=1.0, + ) From 4f38a9e80a095026e38271a0fd60c18eee620f7b Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Fri, 4 Sep 2026 10:56:59 -0700 Subject: [PATCH 08/11] fix(container): arm the heartbeat watchdog only inside the container `_run-task-internal` started its host-heartbeat watchdog as an unconditional side effect of the command body. That thread's whole authority is `os._exit(137)`, and the only process it may reap that way is the container's own disposable main -- there is no container to orphan anywhere else. A test invokes the command in-process, legitimately: the command must refuse a malformed context.json, and proving that means calling it. The pytest worker inherited the thread, which found no heartbeat and exited the worker 40s later (20s grace + 20s stale window), inside whatever unrelated test file that worker had since moved on to. Every property of the failure came from the missing guard: it named a different test on each run and on each platform (opencode on Linux, sandbox_record_cli on Windows), carried no traceback because there is no exception to raise, and hid at high parallelism -- with 14 local workers the run ended before the timer fired, so it reproduced only on CI's 2. It also took the coverage gate with it: a dead worker returns no coverage data, so one killed process reported as "total of 65.13 is less than fail-under=80.00", naming neither the test nor the cause. Timing on both platforms is exactly 40s from that test to the worker's death. The watchdog is now defined and started only under CODER_EVAL_IN_CONTAINER, which docker_runner sets on the container's argv -- not on `driver`, since this same command rewrites `driver: docker` -> `tempdir` before building the in-container Orchestrator and a driver-based gate would disarm itself on exactly the path that needs it. CE052 makes it permanent: an `os._exit` in src/ must sit inside a branch testing that var. Its rule test asserts the real module passes, so the rule cannot pass vacuously. The behavioural test asserts on the live thread list rather than by patching `threading`, and fails when the guard is inverted. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 2 +- pyproject.toml | 1 + .../cli/run_task_internal_command.py | 109 +++++++++++------- ..._process_lethal_must_be_container_gated.py | 91 +++++++++++++++ tests/lint/runner.py | 2 + tests/test_custom_lint.py | 54 +++++++++ tests/test_detached_grading_boundaries.py | 21 ++++ 7 files changed, 236 insertions(+), 44 deletions(-) create mode 100644 tests/lint/rules/ce052_process_lethal_must_be_container_gated.py diff --git a/CLAUDE.md b/CLAUDE.md index 1474465a..92e4139f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -223,7 +223,7 @@ make plugin-reference # the plugin's bundled criteria reference from the models Editing `src/coder_eval/pricing.py` means editing `evalboard/lib/pricing.ts` too — it is a hand-copied mirror, and `evalboard/lib/__tests__/pricing-parity.test.ts` fails the build on drift in either direction. -Recent additions, each traceable to a shipped defect: **CE047** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE049** (never coalesce a possibly-unmeasured score to a numeric literal — `score or 0.0` publishes "measured and scored zero" while meaning "never measured", which is how an ungraded night reached four unfiltered `avg(Score)` dashboards as a real zero), **CE050** (no untyped `getattr` probe for a discriminated-union field — pyright cannot see the string, so a rename degrades the guard to a permanent no-op; scoped to criterion-shaped receivers because `command`/`tool`/`prompt` are far too common to flag on their own), **CE051** (a sandbox driver may not be rewritten silently — the driver IS the isolation boundary, so a downgrade must be an explicit, stamped, operator-visible decision), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's). +Recent additions, each traceable to a shipped defect: **CE047** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE049** (never coalesce a possibly-unmeasured score to a numeric literal — `score or 0.0` publishes "measured and scored zero" while meaning "never measured", which is how an ungraded night reached four unfiltered `avg(Score)` dashboards as a real zero), **CE050** (no untyped `getattr` probe for a discriminated-union field — pyright cannot see the string, so a rename degrades the guard to a permanent no-op; scoped to criterion-shaped receivers because `command`/`tool`/`prompt` are far too common to flag on their own), **CE051** (a sandbox driver may not be rewritten silently — the driver IS the isolation boundary, so a downgrade must be an explicit, stamped, operator-visible decision), **CE052** (an `os._exit` must sit inside a branch testing `CODER_EVAL_IN_CONTAINER` — it is the right primitive only for reaping the container's own disposable main process, and `run_task_internal_command` armed its heartbeat watchdog, a daemon thread whose whole authority is `os._exit(137)`, unconditionally: a test that invoked the command in-process left the pytest worker holding that thread, which exited the worker 40s later inside an unrelated test file, naming a different test on each run and on each platform with no traceback — and the dead worker's lost coverage data then failed the gate as `65.13 < 80.00`, naming neither the test nor the cause), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's). When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033–CE036 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE036 enforces the `live_verdict` determinism + monotonicity contract (`criteria/base.py`) that `EarlyStopWatcher`'s latching, deferred fail-stop, and flip-attribution silently depend on: monotonicity over arbitrary Python is undecidable, so instead of a static check it REPLAYS each live criterion against every prefix of recorded trajectories (`tests/lint/live_verdict_contract.py::CASES`) — on the authored ordering AND under seeded shuffles (`permuted_violations`, which catch order-sensitive bugs the authored walk misses) — and asserts the property directly, plus registry-derived coverage — every `LiveSuccessCriterion` in the union must have cases, and every polarity its instances claim via `live_decidable_polarities()` must actually be reached by one (otherwise a single always-`undecided` fixture would "cover" a type while proving nothing). Adding a live criterion therefore means adding `ContractCase`s in the same change. CE035 resolves every `steps..outputs.` / `needs..outputs.` reference in `.github/workflows/**` to a writer that actually produces that key — GitHub expands an unwritten output to the empty string, so a typo degrades a gate silently and actionlint models `steps.*.outputs` as an open string map. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.) diff --git a/pyproject.toml b/pyproject.toml index a5974a2d..b8978938 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -229,6 +229,7 @@ external = [ "CE049", "CE050", "CE051", + "CE052", ] # custom architectural lint rules (tests/lint/) [tool.ruff.lint.pylint] diff --git a/src/coder_eval/cli/run_task_internal_command.py b/src/coder_eval/cli/run_task_internal_command.py index 62f176fe..bcee7c28 100644 --- a/src/coder_eval/cli/run_task_internal_command.py +++ b/src/coder_eval/cli/run_task_internal_command.py @@ -91,50 +91,73 @@ def run_task_internal_command( import threading import time - def _watch_host_heartbeat() -> None: - heartbeat = output_dir / HEARTBEAT_FILENAME - # Grace period for the host to write the first counter value. - time.sleep(HEARTBEAT_STALE_SECONDS) - last_counter = "" - last_mtime = 0.0 - last_change = time.monotonic() - while True: - try: - current = heartbeat.read_text(encoding="utf-8") - except (FileNotFoundError, OSError): - current = "" - try: - current_mtime = heartbeat.stat().st_mtime - except (FileNotFoundError, OSError): - current_mtime = 0.0 - now = time.monotonic() - if heartbeat_is_alive(current, last_counter, current_mtime, last_mtime): - last_counter = current - last_mtime = current_mtime - last_change = now - if now - last_change > HEARTBEAT_STALE_SECONDS: - logger.error( - "Host heartbeat stale (>%ss); exiting to reap orphan container.", - HEARTBEAT_STALE_SECONDS, - ) - # os._exit skips atexit and IO flushing, so the error line - # above would routinely be lost -- making a genuine - # stale-heartbeat suicide indistinguishable from an external - # SIGKILL in the archived logs. Flush best-effort first; - # never let a flush failure stop the exit. - import sys as _sys - - for _handler in logging.getLogger().handlers: + # ARMED ONLY INSIDE THE CONTAINER, and not even defined outside one. The + # watchdog's whole authority is `os._exit(137)` on the process it runs in, + # and the only process that may be reaped that way is the container's own + # disposable main -- there is no container to orphan anywhere else, so + # outside one the thread can do nothing but harm. It did: a test invoked + # this command in-process (legitimately -- the command must refuse a + # malformed context.json, and proving that means calling it) and the pytest + # worker inherited the thread, which found no heartbeat and 40s later exited + # the worker mid-way through an unrelated test file. It named a different + # test on each run and on each platform, carried no traceback, and took that + # worker's coverage data with it -- so the gate reported "65.13 < 80.00", + # naming neither the test nor the cause. + # + # Gated on CODER_EVAL_IN_CONTAINER (set by docker_runner on the container's + # argv), NOT on `driver`, for the same reason the reference-permission + # window is: this command rewrites `driver: docker` -> `tempdir` before + # building the in-container Orchestrator, so a driver-based gate would + # disarm itself on exactly the path that needs it. See + # `Sandbox.enforces_permission_windows`. + if _os.environ.get("CODER_EVAL_IN_CONTAINER") == "1": + + def _watch_host_heartbeat() -> None: + heartbeat = output_dir / HEARTBEAT_FILENAME + # Grace period for the host to write the first counter value. + time.sleep(HEARTBEAT_STALE_SECONDS) + last_counter = "" + last_mtime = 0.0 + last_change = time.monotonic() + while True: + try: + current = heartbeat.read_text(encoding="utf-8") + except (FileNotFoundError, OSError): + current = "" + try: + current_mtime = heartbeat.stat().st_mtime + except (FileNotFoundError, OSError): + current_mtime = 0.0 + now = time.monotonic() + if heartbeat_is_alive(current, last_counter, current_mtime, last_mtime): + last_counter = current + last_mtime = current_mtime + last_change = now + if now - last_change > HEARTBEAT_STALE_SECONDS: + logger.error( + "Host heartbeat stale (>%ss); exiting to reap orphan container.", + HEARTBEAT_STALE_SECONDS, + ) + # os._exit skips atexit and IO flushing, so the error line + # above would routinely be lost -- making a genuine + # stale-heartbeat suicide indistinguishable from an external + # SIGKILL in the archived logs. Flush best-effort first; + # never let a flush failure stop the exit. + import sys as _sys + + for _handler in logging.getLogger().handlers: + with contextlib.suppress(Exception): + _handler.flush() with contextlib.suppress(Exception): - _handler.flush() - with contextlib.suppress(Exception): - _sys.stdout.flush() - with contextlib.suppress(Exception): - _sys.stderr.flush() - _os._exit(137) - time.sleep(HEARTBEAT_STALE_SECONDS / 4) - - threading.Thread(target=_watch_host_heartbeat, daemon=True).start() + _sys.stdout.flush() + with contextlib.suppress(Exception): + _sys.stderr.flush() + _os._exit(137) + time.sleep(HEARTBEAT_STALE_SECONDS / 4) + + threading.Thread(target=_watch_host_heartbeat, daemon=True).start() + else: + logger.debug("Not in a container; host-heartbeat watchdog not armed.") task_yaml = input_dir / "task.yaml" context_json = input_dir / "context.json" diff --git a/tests/lint/rules/ce052_process_lethal_must_be_container_gated.py b/tests/lint/rules/ce052_process_lethal_must_be_container_gated.py new file mode 100644 index 00000000..a05e670c --- /dev/null +++ b/tests/lint/rules/ce052_process_lethal_must_be_container_gated.py @@ -0,0 +1,91 @@ +"""CE052: a process-lethal call must be gated on actually being in the container. + +``os._exit`` bypasses ``atexit``, buffered IO, ``finally`` blocks and every +exception handler: the process is simply gone. That is the correct primitive for +exactly one thing in this codebase — reaping the container's own disposable main +process when the host that started it has died — and it is safe there only +because that process is *ours to destroy*. In any other process it is not a +degraded outcome, it is an unattributable one. + +The motivating bug: ``run_task_internal_command`` armed its host-heartbeat +watchdog — a daemon thread whose whole authority is ``os._exit(137)`` — as an +unconditional side effect of the command body. A test invoked that command +in-process (legitimately: the command must refuse a malformed ``context.json``, +and asserting that means calling it), and the pytest worker inherited the +thread. Forty seconds later — 20s grace plus the 20s stale window — it found no +heartbeat and exited the worker, mid-way through whatever unrelated test file +that worker had since moved on to. + +Every property of that failure is the one this rule exists to prevent: + + * it named the wrong test — a different one on each run, on each platform, + with no traceback, because there is no exception to raise; + * it was invisible at low load — with 14 local workers the file finished and + the run ended before the timer fired, so it reproduced only on CI's 2; + * and it took the coverage gate with it. A dead worker returns no coverage + data, so a single killed process reported as "total of 65.13 is less than + fail-under=80.00" — a failure naming neither the test nor the cause. + +Fires on ``os._exit(...)`` anywhere in ``src/coder_eval/`` that is not lexically +inside a branch testing ``CODER_EVAL_IN_CONTAINER``. That env var is the repo's +established in-container predicate (``Sandbox.enforces_permission_windows``, +``orchestration/evaluation.resolve_reference_dir``) and is deliberately NOT +``sandbox.driver`` — ``run_task_internal_command`` rewrites the driver to +``tempdir`` before building the in-container Orchestrator, so a driver-based gate +disables itself on precisely the path that needs it. + +The check is lexical (an enclosing ``if``/``elif`` whose test mentions the var), +not a data-flow proof. That is enough to force the guard to be written down at +the site, which is the property that was missing. +""" + +import ast +import re + +from tests.lint.rules.base import BaseRule + + +_GATE = "CODER_EVAL_IN_CONTAINER" + +_MESSAGE = ( + "`os._exit` here is not gated on CODER_EVAL_IN_CONTAINER. It kills the process outright — " + "no atexit, no finally, no traceback — which is the right primitive ONLY for the container's " + "own main process. Anywhere else it destroys a host process that merely called this code: an " + "unconditionally-armed watchdog once exited a pytest worker 40s after the test that armed it, " + "reporting as a random crash in an unrelated file and as a bogus coverage failure. Gate it on " + '`os.environ.get("CODER_EVAL_IN_CONTAINER") == "1"`, or add `# noqa: CE052` with a reason.' +) + + +def _is_os_exit(node: ast.Call) -> bool: + """``os._exit(...)`` under any module alias (it is imported as ``_os`` here).""" + return isinstance(node.func, ast.Attribute) and node.func.attr == "_exit" + + +class ProcessLethalMustBeContainerGated(BaseRule): + id = "CE052" + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + # `(^|sep)` so a repo-relative path is in scope too; see CE047. + self._in_scope = bool(re.search(r"(?:^|[/\\])src[/\\]coder_eval[/\\]", filepath)) + # Tests of enclosing `if`/`elif` statements, innermost last. + self._guards: list[ast.expr] = [] + + def visit_If(self, node: ast.If) -> None: + # Only the body is guarded — the `else` arm is the ungated branch, which + # is exactly where an inverted guard would put the lethal call. + self._guards.append(node.test) + for child in node.body: + self.visit(child) + self._guards.pop() + for child in node.orelse: + self.visit(child) + + def visit_Call(self, node: ast.Call) -> None: + if self._in_scope and _is_os_exit(node) and not self._container_gated(): + self.violation(node, _MESSAGE) + self.generic_visit(node) + + def _container_gated(self) -> bool: + return any(_GATE in ast.dump(test) for test in self._guards) diff --git a/tests/lint/runner.py b/tests/lint/runner.py index f302da08..8bfb6846 100644 --- a/tests/lint/runner.py +++ b/tests/lint/runner.py @@ -31,6 +31,7 @@ from tests.lint.rules.ce049_no_score_or_zero import NoScoreOrZero from tests.lint.rules.ce050_no_union_getattr_probe import NoUnionGetattrProbe from tests.lint.rules.ce051_no_driver_override import NoDriverOverride +from tests.lint.rules.ce052_process_lethal_must_be_container_gated import ProcessLethalMustBeContainerGated from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore @@ -85,6 +86,7 @@ NoScoreOrZero, NoUnionGetattrProbe, NoDriverOverride, + ProcessLethalMustBeContainerGated, ] # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 9583e945..2c65a278 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -3764,3 +3764,57 @@ def test_is_out_of_scope_in_the_model_module(self): 'cfg.model_copy(update={"driver": "tempdir"})', filepath="src/coder_eval/models/sandbox.py", ) + + +class TestCE052ProcessLethalMustBeContainerGated: + """CE052 flags an `os._exit` that is not gated on being in the container.""" + + @staticmethod + def _run(src: str, filepath: str = "src/coder_eval/cli/run_task_internal_command.py"): + import ast + + from tests.lint.rules.ce052_process_lethal_must_be_container_gated import ( + ProcessLethalMustBeContainerGated, + ) + + return ProcessLethalMustBeContainerGated(filepath).check(ast.parse(src)) + + def test_flags_an_ungated_exit(self): + assert self._run("os._exit(137)") + + def test_flags_it_under_an_unrelated_guard(self): + assert self._run("if stale:\n os._exit(137)") + + def test_flags_it_in_the_else_arm_of_the_container_guard(self): + """An inverted guard is the shape a well-meaning refactor produces.""" + src = 'if os.environ.get("CODER_EVAL_IN_CONTAINER") == "1":\n pass\nelse:\n os._exit(137)' + assert self._run(src) + + def test_allows_a_gated_exit(self): + src = 'if os.environ.get("CODER_EVAL_IN_CONTAINER") == "1":\n os._exit(137)' + assert not self._run(src) + + def test_allows_it_nested_deeper_inside_the_gate(self): + """The real site defines a function and a loop inside the guard.""" + src = ( + 'if _os.environ.get("CODER_EVAL_IN_CONTAINER") == "1":\n' + "\n" + " def _watch() -> None:\n" + " while True:\n" + " if stale:\n" + " _os._exit(137)\n" + ) + assert not self._run(src) + + def test_is_out_of_scope_outside_the_package(self): + assert not self._run("os._exit(137)", filepath="scripts/reap.py") + + def test_the_real_module_is_clean(self): + """The rule must actually pass on the site it was written for — a rule + that only ever fires on synthetic input proves nothing about the tree.""" + from pathlib import Path + + path = Path("src/coder_eval/cli/run_task_internal_command.py") + source = path.read_text(encoding="utf-8") + assert not self._run(source, filepath=str(path)) + assert "_os._exit(137)" in source, "the guarded call must still exist" diff --git a/tests/test_detached_grading_boundaries.py b/tests/test_detached_grading_boundaries.py index 278297d1..3c5bb5af 100644 --- a/tests/test_detached_grading_boundaries.py +++ b/tests/test_detached_grading_boundaries.py @@ -168,6 +168,27 @@ def _run_with_context(tmp_path: Path, grade: object): ["_run-task-internal", "--input", str(input_dir), "--output", str(tmp_path / "out")], ) + def test_invoking_the_command_here_arms_no_process_lethal_watchdog(self, tmp_path: Path) -> None: + """The command's heartbeat watchdog reaps an ORPHANED CONTAINER by calling + `os._exit(137)` on itself. This suite invokes the command in-process, so an + unconditionally-armed thread exits the pytest WORKER instead — 40s later + (20s grace + 20s stale), inside whatever unrelated test that worker has + moved on to. It shipped that way: it killed a different test on each run + and on each platform, with no traceback, and the dead worker's lost + coverage data then failed the gate as `65.13 < 80.00`. + + Asserted on the live thread list rather than by patching `threading`, so + the guard is proven at the only place that matters — whether a thread now + exists in this process. + """ + import threading + + before = {t.name for t in threading.enumerate()} + self._run_with_context(tmp_path, True) + leaked = [t for t in threading.enumerate() if t.name not in before and t.daemon] + + assert not leaked, f"the command armed a process-lethal daemon thread outside a container: {leaked}" + def test_a_non_boolean_grade_is_a_hard_error(self, tmp_path: Path) -> None: """A hand-edited or older-format `"grade": "false"` is a truthy str typed as bool, which would silently grade a run that asked not to be.""" From 19c60e46fee520382941b28233ec6b5bf4363b4d Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Fri, 4 Sep 2026 12:25:31 -0700 Subject: [PATCH 09/11] fix(execute): close the verdict-divergence, trust-gate and fabricated-rate defects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the second review of PR #154 (against 7d5d55d). Every fix is a place where the code substituted something plausible for something it did not know. Verdict parity — `run` must equal `execute` + `evaluate`: - `_terminal_status` put `max_turns_exhausted` ABOVE the grading switch, so an execute row finalized MAX_TURNS_EXHAUSTED. That status is an execution fact, so the first arm then pinned it forever: identical agent output scored SUCCESS/1.0 under `run` and MAX_TURNS_EXHAUSTED under `execute` -> `evaluate`. It is not knowable without grading (`run` returns SUCCESS when the criteria pass), so the fact is carried on the row and the status is left to the grade. - `partition_for_resume` routed on FinalStatus.category, so an execute row that also tripped a run limit (TIMEOUT, a budget stop) was called "already complete" and stayed permanently unscored -- while `evaluate ` graded the identical bytes. The test is now the row's evidence: executed, never scored. - `evaluate` read `final_status` as this pass's own outcome. A preserved TIMEOUT exited 0 under "All criteria passed" (a CI wrapper went green on a failed row); a preserved ERROR printed the original run's crash message as though grading had crashed, claimed the row was left ungraded (false), and discarded a verdict just computed at 1.000. Trust boundary: - The recorded-config gate walked only success_criteria + hooks, so a shared run dir whose criteria were all file_exists passed it and still reached `uv pip install` / `npm install` / `git clone` with recorded values. The scan now covers sandbox provisioning and llm_judge; `git clone` gets a `--` separator (the URL sits in argv position 2). - `Sandbox.resolve_files` joined a criterion path onto the sandbox root with no containment check, and `Path(root) / '/etc/passwd'` is `/etc/passwd`. It was the one task-authored path skipping `_resolve_within_sandbox` -- defensible until `evaluate ` began rebuilding criteria from a shareable artifact. - `_write_synthetic_task_json` was the one writer of task.json not routed through `write_text_atomic`, so it followed a symlink at its temp name. - `_assert_grade_honored` refused in memory only, leaving the graded record on disk for `execute --resume` and `aggregate` to re-absorb; it now quarantines to a `.graded` sidecar and keys on evidence rather than on the status label. write_text_atomic, both halves: - A fixed temp name plus O_EXCL turned a leftover from a SIGKILL into a permanent refusal to persist the record -- and `--resume` then re-ran the task into the same run dir and hit it again, re-paying for the agent every pass. The name is now unique per call; O_EXCL keeps its guarantee. - Creating it 0600 made every container-written task.json unreadable by the host across the docker bind mount on Linux (an unguarded read). Mode is 0666 so the umask applies, as `Path.write_text` did. Fabricated rates: - `tasks_graded` keeps ERROR rows, correct under `run` but not under `execute`, where nothing was measured at all: a 100-task execute night with 5 crashes published pass_rate 0.0 / error_share 1.0. Both are None when no row produced a verdict. - evalboard `turnBudgetRateForTasks` compared a raw "SUCCESS", booking an ungraded row as a budget miss; watchlist `attention()` scored an all-ungraded skill failRate 1.0 and put it top of an exec-triage hero. The remaining raw literals are converted to the typed helpers, and trends paints ungraded grey rather than red. Enforcement: - CE053: no bare run-record filename literal outside path_utils. The constant shipped with a rename-safety rationale while twelve literals stayed behind, including all three rglob sites its own comment cites; those are migrated. - `[tool.ruff.lint] external` is completed and now has a parity test -- CE047 and CE048 advertise `# noqa` codes ruff was rejecting with RUF102. Also: `graded_on_host` and `replicate_index` on evaluate's non-delegating branch; `run --allow-host-grading` without `--resume` is a BadParameter instead of a silent no-op; context.json's variant_id/replicate_index are validated, not just annotated; `_pick_worst_status`'s priority map is typed and indexed directly; `_skip_hooks_for_adopted` takes the command list instead of a magic string; the simulation grade=False stub raises instead of guaranteeing a downstream ValueError. Tests: the evalboard's new denominators (trends/watchlist/overview) had zero assertions; `_seed_from_prior_result`'s sensor compared `iterations` and `simulation` at their defaults, so it passed with the carry line deleted, and now asserts anti-vacuity first; the two `context.get("grade", True)` source greps are replaced by a behavioural test that patches Orchestrator. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/commands/coder-eval-review.md | 3 +- CLAUDE.md | 17 +- docs/REPORT_SCHEMA.md | 10 +- docs/USER_GUIDE.md | 14 +- docs/tutorials/02-ci-pipeline.md | 11 +- evalboard/app/trends/trends-view.tsx | 11 +- evalboard/lib/__tests__/overview.test.ts | 79 ++++++ evalboard/lib/__tests__/trends.test.ts | 27 ++ evalboard/lib/__tests__/watchlist.test.ts | 32 +++ evalboard/lib/overview.ts | 14 +- evalboard/lib/status.ts | 23 +- evalboard/lib/trends.ts | 4 +- evalboard/lib/watchlist.ts | 15 +- pyproject.toml | 26 ++ src/coder_eval/cli/evaluate_command.py | 98 ++++++- src/coder_eval/cli/report_command.py | 3 +- src/coder_eval/cli/run_command.py | 10 + .../cli/run_task_internal_command.py | 104 ++++--- src/coder_eval/isolation/docker_runner.py | 74 +++-- src/coder_eval/models/experiment.py | 5 +- src/coder_eval/models/results.py | 31 +- src/coder_eval/orchestration/batch.py | 41 ++- src/coder_eval/orchestration/experiment.py | 24 +- src/coder_eval/orchestration/regrade.py | 64 ++++- src/coder_eval/orchestrator.py | 97 +++++-- src/coder_eval/path_utils.py | 48 +++- src/coder_eval/reports.py | 17 +- src/coder_eval/reports_junit.py | 7 +- src/coder_eval/reports_stats.py | 4 +- src/coder_eval/sandbox.py | 45 ++- .../ce053_run_record_filename_literal.py | 70 +++++ tests/lint/runner.py | 2 + tests/test_custom_lint.py | 83 ++++++ tests/test_detached_grading_boundaries.py | 266 ++++++++++++++++-- tests/test_execute_command.py | 18 +- tests/test_execute_evaluate_loop.py | 62 +++- tests/test_experiment_reports.py | 99 +++++++ tests/test_reports.py | 57 ++++ tests/test_resume.py | 49 +++- tests/test_seed_from_prior_result.py | 22 ++ 40 files changed, 1468 insertions(+), 218 deletions(-) create mode 100644 tests/lint/rules/ce053_run_record_filename_literal.py diff --git a/.claude/commands/coder-eval-review.md b/.claude/commands/coder-eval-review.md index 7f53b0cd..d778b1f8 100644 --- a/.claude/commands/coder-eval-review.md +++ b/.claude/commands/coder-eval-review.md @@ -24,7 +24,8 @@ The run layout (`runs/////…`, `` a zero-p 1. Read `/run.json` if present (for context — `run_id`, `start_time`). 2. Glob `/*/*/*/task.json` and read each one. 3. Read `/analysis.md` if present — it already diagnoses many failures; lean on its findings rather than re-deriving them. -4. A task counts as **failed** if `final_status != "SUCCESS"` **or** `weighted_score < 0.9`. Skip passing tasks for now (we may extend to passing tasks later — the schema supports it). +4. Skip any task whose `final_status` is `"NOT_GRADED"` — `coder-eval execute` produced it, no criterion ran, and `weighted_score` is `null`. It is neither a pass nor a failure, and comparing `null < 0.9` would book every ungraded row as a failure to review. +5. Of the rest, a task counts as **failed** if `final_status != "SUCCESS"` **or** `weighted_score < 0.9`. Skip passing tasks for now (we may extend to passing tasks later — the schema supports it). If no `task.json` files exist, write an empty `review_index.json` (`{"reviews": []}`) and exit. diff --git a/CLAUDE.md b/CLAUDE.md index 92e4139f..aba27c48 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,6 +27,13 @@ coder_eval/ ├── fs_permissions.py # set_permissions: stacked chmod window (via Sandbox.set_permissions) ├── pricing.py # Model pricing / cost calculation (ModelPricing, calculate_cost, register_pricing) ├── litellm_cost.py # Join proxy-captured ACTUAL per-call cost/cache onto turns (LiteLLM backend; apply_actual_cost) +├── reports_html.py # Single-file HTML report (the evalboard's static twin) +├── reports_stats.py # Shared report statistics + the ungraded rendering helpers (format_score, is_env_table_key) +├── formatting.py # Human-readable number/duration formatting shared by the renderers +├── invocation_log.py # JSON Lines CLI-invocation log the `cli_called` criterion reads +├── telemetry.py # App Insights / OpenTelemetry emission (CoderEval.Task.End et al.) +├── isolation/ # driver: docker — docker_runner.py builds, runs and reaps one container per task +├── optimize/ # Prompt/config optimization helpers ├── utils.py # Version info helpers │ ├── agents/ @@ -96,6 +103,8 @@ coder_eval/ │ ├── evaluate_command.py # `coder-eval evaluate` (grade a dir, or re-grade a run dir) + `run_evaluation` │ ├── evaluate_target.py # PURE shape detection for evaluate's positionals (run dir ⟺ holds task.json) │ ├── report_command.py # `coder-eval report` +│ ├── aggregate_command.py # `coder-eval aggregate` — rebuild run.json from the task.json rows on disk (the step right after `execute`) +│ ├── run_task_internal_command.py # `_run-task-internal` — the in-container entry point DockerRunner invokes; never called by a user │ ├── run_helpers.py # CLI helper functions │ ├── console.py # Rich console instance │ └── utils.py # CLI utilities @@ -150,9 +159,9 @@ action.yml # Published composite GitHub Action (coder-ev - **Reference solutions are directory-only, and shielded (partially) from the agent**: `task.reference` is a single required `directory:` (relative to the task YAML) — the inline `code:` / single-file `file:` forms are gone, because a directory is the only shape that can be permission-gated as a unit; a `model_validator(mode="before")` gives the removed forms a migration error. The orchestrator stages a **per-run private copy** (`orchestration/evaluation.py::stage_reference_dir`, symlinks stripped) into a tempdir, removed in `_cleanup` via `path_utils.rmtree_restrictive` (keyed on `_reference_staging_root`, recorded BEFORE the copy so a failed copy still cleans up; `rmtree(ignore_errors=True)` silently declines on a tree left at 000) and deliberately never preserved into `run_dir/artifacts`. That copy is held at mode `000` for the whole of every `agent.communicate` call via **`Sandbox.set_permissions`**, the driver-aware wrapper over `fs_permissions.py::set_permissions`. Windows **stack**: exiting restores the *enclosing* window's mode, only the outermost exit restores the pre-window mode — that is what makes a mid-turn re-grant (`mode=READ_ONLY_MODE`) expressible, and it covers two windows at the same mode so no refcount is needed. The window is enforced **only inside a docker container** (`Sandbox.enforces_permission_windows`) and is a no-op on the host, where the agent shares our uid. **That gate keys on the `CODER_EVAL_IN_CONTAINER` env var, NOT `sandbox.driver`** — `run_task_internal_command` rewrites `driver: docker` → `tempdir` before building the in-container Orchestrator, so a driver-based gate would silently disable the anti-cheat on exactly the path that needs it (regression-guarded by `TestSandboxDriverGate`); `resolve_reference_dir` gates its `/work/references` branch on the same var for the same reason. The task directory is **not** shielded (`:ro` mount → EROFS, and the same YAML is readable at `/work/input`). Criteria address reference files with the `$REFERENCE_DIR` token (same resolver as `$TASK_DIR`) and the `REFERENCE_DIR` env var for `run_command`; `reference_comparison` names one file via `reference_file`. Docker mounts a throwaway **read-write** copy at `/work/references` (a `:ro` mount cannot be chmod'd — EROFS), masks the in-task-dir original with an empty tmpfs, and drops `DAC_OVERRIDE`/`DAC_READ_SEARCH`. `FOWNER`/`CHOWN` are deliberately **NOT** dropped: the in-container orchestrator that applies the window is the same root process with the same caps, so dropping `FOWNER` breaks *the harness's own* chmod wherever the bind mount preserves a non-root owner (native Linux — verified: `chmod: Operation not permitted`), i.e. exactly where the drop would otherwise bite. A window that cannot be applied is now a hard error, not a warning: `Sandbox.set_permissions` passes `strict=True` whenever it enforces, so an unprotected run fails instead of producing a normal-looking score. **KNOWN GAP — this is defense-in-depth, not a boundary**: (a) `chmod(2)` is gated on owner-or-`CAP_FOWNER` and the container runs as root owning the copy, so a deliberate `chmod 755 /work/references` restores access; (b) the window spans `agent.communicate` only, and nothing reaps agent child processes at turn end, so a backgrounded read loop succeeds once the window closes. The **write** half of (b) is closed — `path_utils.digest_tree` hashes the tree at staging and `Orchestrator._verify_reference_integrity` re-checks before grading, raising `ReferenceTamperedError` (→ `FinalStatus.ERROR`) on a mismatch so an agent cannot overwrite the reference to drive `reference_comparison` to 1.0. Passive reads are blocked; an adversarial agent is not. Full containment requires running the agent as a non-root uid AND holding the window for the agent's whole lifetime — follow-up. `tasks/anti_cheat_reference` probes the passive-read half. - **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex and Antigravity (both run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced), and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. Full table + rationale: docs/agents/HARNESS_PARITY.md. - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly. -- **Execute vs. run (the grading switch)**: `coder-eval execute` is `coder-eval run` with grading removed — the agent runs and the full trajectory is captured, but no criterion is checked, `weighted_score` is `None` (never `0.0`, which would be indistinguishable from "graded and scored zero"), and the row finalizes as **`FinalStatus.NOT_GRADED`**, whose `category` is a **fourth** bucket, `"ungraded"`. Ungraded rows leave BOTH sides of every rate: `RunSummary.pass_rate` / `error_share` and `VariantAggregate.pass_rate` divide by `tasks_graded` (`tasks_run - tasks_not_graded`), and `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter. **Only SUCCESS/FAILURE collapse into it** — `ERROR`, `TIMEOUT`, `BUILD_FAILED`, `MAX_TURNS_EXHAUSTED` and the budget stops are facts about the *run*, not about grading, and still apply (so `execute` still exits non-zero on a crash). The switch is `BatchRunConfig.grade` → `Orchestrator(grade=...)` → the **four** grading call sites (single-shot, evaluate-only, the simulation dialog check, and post-failure diagnostics); it crosses the docker boundary in `context.json` (defaulting to `True` in-container, so a host predating `execute` keeps grading). It is **deliberately not a task-config field** — no 5-layer merge, no `-D` path — because a task YAML must never declare itself ungraded; only the invoking command decides. `run` and `execute` share one body (`run_command.run_pipeline`) and differ solely in that flag, so there is no third code path. Two things are refused rather than degraded: `--junit-xml` (a report of verdicts, and there are none — though `reports_junit` still emits `` for an ungraded row it encounters) and simulation tasks (their turn-continuation logic reads criteria results, so an ungraded dialog would silently change its own stopping behavior). `stop_early:` blocks are inert under `execute` for the same reason the kill switch exists: the full trajectory is the deliverable. Motivating consumer: an external harness (Harbor / Terminal-Bench 2.0) that builds its own container, calls coder-eval as the agent, and grades with its own tests. -- **Detached grading (`evaluate` over a run dir) + `Sandbox.adopt`**: `coder-eval evaluate` takes two shapes, told apart by a **pure** resolver (`cli/evaluate_target.py`) on one probe — a target holding `task.json` is a run directory. Run-dir mode rebuilds the task from the run's own `task_config.resolved`, **not** by re-loading the YAML: `resolved` is post-merge, so variant overrides / `-D` / dataset expansion are already baked in and re-loading the source would silently grade a *different* task (fallback to `source_file` only when `resolved` no longer validates, and loudly). It seeds the fresh result from the prior one via `Orchestrator(prior_result=...)` → `_seed_from_prior_result`, which carries the trajectory (every derived figure — tokens, cost, `command_stats`, `model_used` — recomputes from `iterations`), `iteration_count`, execution facts, and **`early_stop` — load-bearing, because gate selection is FIRED-ONLY**: dropping it re-grades a truncated trajectory under the full-run strict-AND gate and can flip the verdict. Carrying it is only half the fix — **both** grading paths select the gate through the single `Orchestrator._select_gate()`; the evaluate-only branch a detached grade actually takes originally called `all_criteria_passed` inline, so the seeded field was written and never read. `tests/test_seed_from_prior_result.py` partitions every `EvaluationResult` field as CARRIED or RECOMPUTED and fails closed on a new one, and asserts the two `_select_gate()` call sites. A prior status that `FinalStatus.is_execution_fact` (TIMEOUT / ERROR / BUILD_FAILED / the budget stops) is **preserved**, never overwritten: grading may only move `NOT_GRADED` to SUCCESS/FAILURE, since it neither repeated nor observed the agent phase. Grader-host `environment_info` is preserved as flat `graded_by_*` scalars rather than overwriting the run's (flat, not a nested sub-dict: `environment_info` is rendered as a flat map by the HTML report and typed as one by the evalboard, so a nested capture prints as a Python dict repr). The route recorder follows the same rule: on a detached grade it writes `graded_by_api_routing` / `graded_by_eval_routing` and leaves the run's `api_routing` alone — writing in place contradicted the "prior wins" contract and left a self-contradictory record (a direct route named beside the run's stale `aws_region`/`bedrock_model`). Two other parity fixes: `command_base_path` is now persisted into `environment_info` by `_sync_sandbox_command_path_with_agent` and restored in the evaluate-only branch (closing the PATH gap that method's docstring already named), and `_join_litellm_actual_cost` **skips** when `prior_result` is set (its join keys on a per-Orchestrator nonce the prior turns never carried, so it would clobber already-correct costs). The verdict is written back into the run's `task.json`, with the pre-grade record kept as `task.execute.json` — that in-place write is what makes plain `coder-eval aggregate ` rebuild a graded `run.json` with **zero** new code. **`Sandbox.adopt(workspace)`** is the grade-in-place primitive: it reuses `setup`'s adoption half but skips every *materializing* step (`_setup_template`, `_generate_cli_recorders`, venv/package installs, the destructive `$HOME` remediation), running only non-mutating derivation (mock-dir `+x`, venv *discovery*, plugin-tools pin); `_cleanup_on_exit` stays False so an adopted tree is never moved or deleted, and `Sandbox.was_adopted` is set — the Orchestrator reads it to SKIP the `pre_run`/`post_run` hooks (`run()` calls them unconditionally with `cwd = sandbox_dir`, and several in-tree tasks stage fixtures there with `cp -a /app/[!.]* "$PWD/"`, which would overwrite the agent's deliverables before the criteria read them) and to KEEP `sandbox_path` in the `PreservationMode.NONE` cleanup arm (an adopted tree survives cleanup, so the path is not stale). The hooks' recorded results are carried from the prior run instead. In-place is **more correct**, not merely faster: `_setup_template` filters the copy through `_should_ignore_template_file`, which drops `node_modules` / `dist` / `build` / `.venv` / `.git`, so on the copy path a criterion like `test -f dist/bundle.js` fails as a *copying artifact* rather than as a verdict (verified: 0.00 "does not exist" on copy vs 1.00 in place). Defaults: in-place for a run dir, copy for a bare work dir (criteria can mutate it and it is the user's own tree); `--in-place`/`--copy` override. `adopt` hard-errors on `driver: docker` (a container workspace is unreachable from the host), and grading a `driver: docker` task is REFUSED outright unless `--allow-host-grading` is passed — the earlier behavior silently rewrote the driver to `tempdir`, which ran a container task's criteria against a host filesystem lacking `/verifier` and the image's toolchain (FAILURE for a trajectory `run` scored 1.0, plus `rm -rf /verifier` unsandboxed on the grading machine) and neutralized `adopt`'s own docker guard; an opted-in row is stamped `graded_on_host` so it is never silently comparable with a container-graded one (lint rule CE051). A re-grade refuses on a `reference_digest` mismatch — the digest is persisted into `environment_info` at staging time by `_stage_reference` (it shipped once as a read with no writer anywhere, so the guard was dead code), and `verify_reference_unchanged` now takes the task file it resolves against and RAISES on a vanished or unresolvable reference instead of returning silently. That comparison digests a STAGED copy of the source, not the raw tree: the recorded digest is taken over the staged copy, which `stage_reference_dir` filters through `REFERENCE_COPY_IGNORE` (`.git`), so digesting the source directly compared two differently-filtered trees and reported a permanent false mismatch for any reference that is a git checkout — exactly the case the ignore list exists for. **The recorded config is untrusted input**: `evaluate ` rebuilds the task from a shareable artifact, so a rebuilt config that carries shell (`run_command` criteria, `agent_judge`, `uipath_eval`, and — only on the `--copy` path, since the in-place path skips them — `pre_run`/`post_run`) is REFUSED unless `--allow-recorded-commands` is passed. A warning is not a control: it prints as the command is already being prepared. Passing the task file explicitly (`evaluate `) also bypasses it, since that config came from the operator. The workspace fallback `artifacts / prior.task_id` is containment-checked like its `sandbox_path` sibling (`task_id` is an unvalidated string, and `"../../.."` joins to a real directory `is_dir()` confirms), `_sanitize_restored_path` drops relative entries (they resolve against the grader's cwd) and anything inside the run dir rather than only the workspace, and `write_text_atomic` opens its temp file `O_EXCL|O_NOFOLLOW` — a pre-planted `task.json.tmp` symlink otherwise bypassed the write-back's destination symlink guard entirely. NOTE the Typer command is a thin wrapper over `run_evaluation(...)`, which has real Python defaults — calling a Typer command function in-process hands unspecified options an `OptionInfo` sentinel, which silently made `in_place=None` truthy. -- **`--resume` is command-relative**: `partition_for_resume(tasks, *, grade)` returns a four-way `ResumePartition` (`to_run` / `to_grade` / `prior_results` / `prior_resolved`), because **"finished" is not absolute — it depends on what the resuming command still owes the task**. A `NOT_GRADED` row carries a final status, so the original "has any final status" test called it complete: right for `execute --resume` (it finished executing), and wrong for `run --resume`, which was asked to grade and would instead report "already complete", grade nothing, and **exit 0**. Under `grade=True` those rows route to `to_grade`, where `_grade_resumed_tasks` runs the criteria against the trajectory and workspace already on disk via `orchestration/regrade.py::regrade_in_place` — reusing the agent spend, which is the entire reason `execute` and `run` are separate. The carve-out is **only** for `NOT_GRADED`: `FAILURE`/`ERROR` stay complete under both commands (resume has never retried failures — delete the task.json), and `clear_rerun_artifacts` deliberately skips `to_grade`, whose artifacts are the very thing being graded. A per-task grading failure is warned, STAMPED onto the folded-back row's `error_message` (the console line alone is not durable), and folded back in with its ORIGINAL ungraded result, so one bad row neither aborts the resume nor vanishes from run.json — and the exit gate counts `tasks_not_graded` **when `grade` is True**, so a `run` that graded nothing exits non-zero instead of telling CI the suite is fine. Under `execute` an ungraded row is the expected outcome and never fails the command. `grade` is in `_FINGERPRINT_DIFF_EXEMPT` because `execute` → `run --resume` is a supported flow, not config drift — and the warning's "already-finalized tasks keep their original-config results" text is actively wrong for it. **`orchestration/regrade.py` is the single implementation** shared by that path and `evaluate`'s run-dir mode, which DELEGATES to `regrade_in_place` rather than restating it (it originally hand-built its own Sandbox + Orchestrator and had already drifted — hardcoding `replicate_index=0`, so every replicate but the first was relabelled — which is exactly how two copies become two verdicts for the same run); it raises plain `RegradeError`, which the CLI wraps, since `orchestration/` must not import the CLI layer (CE004). One fidelity rule it enforces: a re-graded row keeps the **agent run's** `started_at`/`duration_seconds`, not the grading pass's — a 10-minute run re-graded in 2s would otherwise report 2s into `average_duration`, the report tables and the evalboard; the grading cost is preserved separately as `environment_info["grading_duration_seconds"]`. +- **Execute vs. run (the grading switch)**: `coder-eval execute` is `coder-eval run` with grading removed — the agent runs and the full trajectory is captured, but no criterion is checked, `weighted_score` is `None` (never `0.0`, which would be indistinguishable from "graded and scored zero"), and the row finalizes as **`FinalStatus.NOT_GRADED`**, whose `category` is a **fourth** bucket, `"ungraded"`. Ungraded rows leave BOTH sides of every rate: `RunSummary.pass_rate` / `error_share` and `VariantAggregate.pass_rate` divide by `tasks_graded` (`tasks_run - tasks_not_graded`), and `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter. **Only SUCCESS/FAILURE collapse into it** — `ERROR`, `TIMEOUT`, `BUILD_FAILED`, `MAX_TURNS_EXHAUSTED` and the budget stops are facts about the *run*, not about grading, and still apply (so `execute` still exits non-zero on a crash). The switch is `BatchRunConfig.grade` → `Orchestrator(grade=...)` → the **four** grading call sites (single-shot, evaluate-only, the simulation dialog check, and post-failure diagnostics); it crosses the docker boundary in `context.json` (defaulting to `True` in-container, so a host predating `execute` keeps grading). It is **deliberately not a task-config field** — no 5-layer merge, no `-D` path — because a task YAML must never declare itself ungraded; only the invoking command decides. `run` and `execute` share one body (`run_command.run_pipeline`) and differ solely in that flag, so there is no third code path. Three things are refused rather than degraded: `--junit-xml` (a report of verdicts, and there are none — though `reports_junit` still emits `` for an ungraded row it encounters), `--allow-host-grading` (it decides how an ungraded row is GRADED, and `execute` grades nothing), and simulation tasks (their turn-continuation logic reads criteria results, so an ungraded dialog would silently change its own stopping behavior). `stop_early:` blocks are inert under `execute` for the same reason the kill switch exists: the full trajectory is the deliverable. Motivating consumer: an external harness (Harbor / Terminal-Bench 2.0) that builds its own container, calls coder-eval as the agent, and grades with its own tests. +- **Detached grading (`evaluate` over a run dir) + `Sandbox.adopt`**: `coder-eval evaluate` takes two shapes, told apart by a **pure** resolver (`cli/evaluate_target.py`) on one probe — a target holding `task.json` is a run directory. Run-dir mode rebuilds the task from the run's own `task_config.resolved`, **not** by re-loading the YAML: `resolved` is post-merge, so variant overrides / `-D` / dataset expansion are already baked in and re-loading the source would silently grade a *different* task (fallback to `source_file` only when `resolved` no longer validates, and loudly). It seeds the fresh result from the prior one via `Orchestrator(prior_result=...)` → `_seed_from_prior_result`, which carries the trajectory (every derived figure — tokens, cost, `command_stats`, `model_used` — recomputes from `iterations`), `iteration_count`, execution facts, and **`early_stop` — load-bearing, because gate selection is FIRED-ONLY**: dropping it re-grades a truncated trajectory under the full-run strict-AND gate and can flip the verdict. Carrying it is only half the fix — **both** grading paths select the gate through the single `Orchestrator._select_gate()`; the evaluate-only branch a detached grade actually takes originally called `all_criteria_passed` inline, so the seeded field was written and never read. `tests/test_seed_from_prior_result.py` partitions every `EvaluationResult` field as CARRIED or RECOMPUTED and fails closed on a new one, and asserts the two `_select_gate()` call sites. A prior status that `FinalStatus.is_execution_fact` (TIMEOUT / ERROR / BUILD_FAILED / the budget stops) is **preserved**, never overwritten: grading may only move `NOT_GRADED` to SUCCESS/FAILURE, since it neither repeated nor observed the agent phase. **`MAX_TURNS_EXHAUSTED` is deliberately NOT one of them under `execute`**: `_terminal_status` puts the `grade=False` arm ABOVE it, because on the graded path it is subordinate to the verdict — `run` returns SUCCESS for a max-turns trajectory whose criteria pass — so it is not knowable without grading. Consuming it first made it terminal AND permanent (the `is_execution_fact` arm then pinned it), so identical agent output scored SUCCESS/1.0 under `run` and MAX_TURNS_EXHAUSTED under `execute` → `evaluate`. The fact survives on `result.max_turns_exhausted`, which `_seed_from_prior_result` carries, so the detached grade walks the identical chain. The CLI must also branch on WHERE a status came from, not on its value: a preserved TIMEOUT exited 0 under "All criteria passed" (a CI wrapper reading the exit code went green on a row run.json counts as failed), and a preserved ERROR printed the ORIGINAL run's crash message as though grading had crashed, claimed the row was "left ungraded" (false — the restored record still read ERROR), and discarded a verdict just computed at 1.000. Grader-host `environment_info` is preserved as flat `graded_by_*` scalars rather than overwriting the run's (flat, not a nested sub-dict: `environment_info` is rendered as a flat map by the HTML report and typed as one by the evalboard, so a nested capture prints as a Python dict repr). The route recorder follows the same rule: on a detached grade it writes `graded_by_api_routing` / `graded_by_eval_routing` and leaves the run's `api_routing` alone — writing in place contradicted the "prior wins" contract and left a self-contradictory record (a direct route named beside the run's stale `aws_region`/`bedrock_model`). Two other parity fixes: `command_base_path` is now persisted into `environment_info` by `_sync_sandbox_command_path_with_agent` and restored in the evaluate-only branch (closing the PATH gap that method's docstring already named), and `_join_litellm_actual_cost` **skips** when `prior_result` is set (its join keys on a per-Orchestrator nonce the prior turns never carried, so it would clobber already-correct costs). The verdict is written back into the run's `task.json`, with the pre-grade record kept as `task.execute.json` — that in-place write is what makes plain `coder-eval aggregate ` rebuild a graded `run.json` with **zero** new code. **`Sandbox.adopt(workspace)`** is the grade-in-place primitive: it reuses `setup`'s adoption half but skips every *materializing* step (`_setup_template`, `_generate_cli_recorders`, venv/package installs, the destructive `$HOME` remediation), running only non-mutating derivation (mock-dir `+x`, venv *discovery*, plugin-tools pin); `_cleanup_on_exit` stays False so an adopted tree is never moved or deleted, and `Sandbox.was_adopted` is set — the Orchestrator reads it to SKIP the `pre_run`/`post_run` hooks (`run()` calls them unconditionally with `cwd = sandbox_dir`, and several in-tree tasks stage fixtures there with `cp -a /app/[!.]* "$PWD/"`, which would overwrite the agent's deliverables before the criteria read them) and to KEEP `sandbox_path` in the `PreservationMode.NONE` cleanup arm (an adopted tree survives cleanup, so the path is not stale). The hooks' recorded results are carried from the prior run instead. In-place is **more correct**, not merely faster: `_setup_template` filters the copy through `_should_ignore_template_file`, which drops `node_modules` / `dist` / `build` / `.venv` / `.git`, so on the copy path a criterion like `test -f dist/bundle.js` fails as a *copying artifact* rather than as a verdict (verified: 0.00 "does not exist" on copy vs 1.00 in place). Defaults: in-place for a run dir, copy for a bare work dir (criteria can mutate it and it is the user's own tree); `--in-place`/`--copy` override. `adopt` hard-errors on `driver: docker` (a container workspace is unreachable from the host), and grading a `driver: docker` task is REFUSED outright unless `--allow-host-grading` is passed — the earlier behavior silently rewrote the driver to `tempdir`, which ran a container task's criteria against a host filesystem lacking `/verifier` and the image's toolchain (FAILURE for a trajectory `run` scored 1.0, plus `rm -rf /verifier` unsandboxed on the grading machine) and neutralized `adopt`'s own docker guard; an opted-in row is stamped `graded_on_host` so it is never silently comparable with a container-graded one (lint rule CE051). A re-grade refuses on a `reference_digest` mismatch — the digest is persisted into `environment_info` at staging time by `_stage_reference` (it shipped once as a read with no writer anywhere, so the guard was dead code), and `verify_reference_unchanged` now takes the task file it resolves against and RAISES on a vanished or unresolvable reference instead of returning silently. That comparison digests a STAGED copy of the source, not the raw tree: the recorded digest is taken over the staged copy, which `stage_reference_dir` filters through `REFERENCE_COPY_IGNORE` (`.git`), so digesting the source directly compared two differently-filtered trees and reported a permanent false mismatch for any reference that is a git checkout — exactly the case the ignore list exists for. **The recorded config is untrusted input**: `evaluate ` rebuilds the task from a shareable artifact, so a rebuilt config that carries shell (`run_command` criteria, `agent_judge`, `llm_judge`, `uipath_eval`, and — only on the `--copy` path, since the in-place path skips them — `pre_run`/`post_run` **and the sandbox's own provisioning**) is REFUSED unless `--allow-recorded-commands` is passed. The provisioning half was the one the gate originally missed, and the worst: `grading_sandbox_config` carries the recorded `sandbox` block through untouched and the `--copy` branch calls `Sandbox.setup`, which reaches `uv pip install ` / `npm install` / `git clone ` — arbitrary code at install time — so a shared run dir whose criteria were all `file_exists` sailed through a scan that walked only `success_criteria`. `git clone` now passes `--` before the URL (argv position 2, so a value beginning with `-` was parsed as an option). `Sandbox.resolve_files` is containment-checked for the same reason: criterion paths were the one task-authored path skipping `_resolve_within_sandbox`, and `Path(root) / '/etc/passwd'` is `/etc/passwd`. A warning is not a control: it prints as the command is already being prepared. Passing the task file explicitly (`evaluate `) also bypasses it, since that config came from the operator. The workspace fallback `artifacts / prior.task_id` is containment-checked like its `sandbox_path` sibling (`task_id` is an unvalidated string, and `"../../.."` joins to a real directory `is_dir()` confirms), `_sanitize_restored_path` drops relative entries (they resolve against the grader's cwd) and anything inside the run dir rather than only the workspace, and `write_text_atomic` opens its temp file `O_EXCL|O_NOFOLLOW` — a pre-planted `task.json.tmp` symlink otherwise bypassed the write-back's destination symlink guard entirely. NOTE the Typer command is a thin wrapper over `run_evaluation(...)`, which has real Python defaults — calling a Typer command function in-process hands unspecified options an `OptionInfo` sentinel, which silently made `in_place=None` truthy. +- **`--resume` is command-relative**: `partition_for_resume(tasks, *, grade)` returns a four-way `ResumePartition` (`to_run` / `to_grade` / `prior_results` / `prior_resolved`), because **"finished" is not absolute — it depends on what the resuming command still owes the task**. A `NOT_GRADED` row carries a final status, so the original "has any final status" test called it complete: right for `execute --resume` (it finished executing), and wrong for `run --resume`, which was asked to grade and would instead report "already complete", grade nothing, and **exit 0**. The routing test is the row's **evidence** (`weighted_score is None and not success_criteria_results`), not its category: keying on `category == "ungraded"` missed every `execute` row that ALSO carries an execution fact — a TIMEOUT or budget stop aborts before grading, so it lands unscored with category `error`/`failed`, and resume filed it as complete while `evaluate ` graded the identical bytes happily. Under `grade=True` those rows route to `to_grade`, where `_grade_resumed_tasks` runs the criteria against the trajectory and workspace already on disk via `orchestration/regrade.py::regrade_in_place` — reusing the agent spend, which is the entire reason `execute` and `run` are separate. The carve-out is **only** for `NOT_GRADED`: `FAILURE`/`ERROR` stay complete under both commands (resume has never retried failures — delete the task.json), and `clear_rerun_artifacts` deliberately skips `to_grade`, whose artifacts are the very thing being graded. A per-task grading failure is warned, STAMPED onto the folded-back row's `error_message` (the console line alone is not durable), and folded back in with its ORIGINAL ungraded result, so one bad row neither aborts the resume nor vanishes from run.json — and the exit gate counts `tasks_not_graded` **when `grade` is True**, so a `run` that graded nothing exits non-zero instead of telling CI the suite is fine. Under `execute` an ungraded row is the expected outcome and never fails the command. `grade` is in `_FINGERPRINT_DIFF_EXEMPT` because `execute` → `run --resume` is a supported flow, not config drift — and the warning's "already-finalized tasks keep their original-config results" text is actively wrong for it. **`orchestration/regrade.py` is the single implementation** shared by that path and `evaluate`'s run-dir mode, which DELEGATES to `regrade_in_place` rather than restating it (it originally hand-built its own Sandbox + Orchestrator and had already drifted — hardcoding `replicate_index=0`, so every replicate but the first was relabelled — which is exactly how two copies become two verdicts for the same run); it raises plain `RegradeError`, which the CLI wraps, since `orchestration/` must not import the CLI layer (CE004). One fidelity rule it enforces: a re-graded row keeps the **agent run's** `started_at`/`duration_seconds`, not the grading pass's — a 10-minute run re-graded in 2s would otherwise report 2s into `average_duration`, the report tables and the evalboard; the grading cost is preserved separately as `environment_info["grading_duration_seconds"]`. - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `stop_early.decide_within`, deliberately lives on `LiveSuccessCriterion` instead (see below) — the watcher must attribute a decision-step timeout to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express. - **Early stop on criterion (opt-in, per-criterion arming)**: a `stop_early:` block (`StopEarlyPolicy`) on a criterion ends a single-shot run early once the run's **armed** criteria decide the outcome, so a raised `max_turns` isn't wasted on the smoke flavor. The block's PRESENCE is the arming and alone activates the watcher — there is **no run-level master switch**: `run_limits.stop_early: false` is the run-level KILL SWITCH that force-disarms every block (the one-line experiment-variant/`-D` override for an authoritative full run), and `run_limits.stop_early: true` (the removed master arm) is a hard `EarlyStopConfigError` at resolution. The block exists on `LiveSuccessCriterion` only (currently `skill_triggered`, `command_executed` — so arming an unobservable criterion is unrepresentable, a pydantic extra-forbid error). Arming carries one implicit trigger (a native live-fail may fail-stop the run); its keys refine it: `on_pass: stop` (pass-stop the moment the criterion live-passes; default `continue` just latches) and `decide_within: N` (still undecided after N tool-call steps latches an **effective fail**, fed through the same fail-stop rule, reported as `decision_budget_exceeded` — an ordinary weighted fail, NOT a gate-bypassing force-fail; cumulative across retry attempts of the same turn). A trigger whose polarity the instance can't decide (per the abstract, checker-independent `live_decidable_polarities()`, a pure function of the criterion's own fields, paired with the checker's `live_verdict` override by lint rule CE025, a registry-based whole-tree check) is **inert by design** — one dataset-fanned YAML line serves both positive rows (pass/timeout live) and distractor rows (fail live). Verdicts **latch**: once a criterion decides, its `live_verdict` is never polled again. Stop rule is weighted, not strict-boolean: `run_limits.stop_early_gate_threshold` (default `1.0`, reproducing strict-AND behavior exactly) is the minimum weighted score (`Σ weight·score / Σ weight` over the armed subset) required to pass; a fail-stop fires once the armed set's **ceiling** (best case for everything still undecided) can no longer reach the threshold — so a low-weight fail or timeout that can't doom the gate is absorbed and the run continues — and is **deferred while any pass-capable armed criterion is undecided** (a distractor misfire never truncates a positive row's recall signal); a pass-stop fires once the `on_pass: stop` subset's **floor** (worst case) already meets the threshold, and is symmetrically **deferred while any pass-capable armed criterion outside the `on_pass: stop` subset is undecided** (so an early pass never freezes a sibling `on_pass: continue` criterion's signal out of the trajectory). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a kill-switched (`stop_early: false`) run. Driven by `orchestration/early_stop.py::EarlyStopWatcher` (built when `early_stop_active(task)`: ≥1 armed criterion, kill switch not thrown) through the agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. Gating is **FIRED-ONLY**: a run the watcher actually cut gates on the **armed subset** via the weighted `EvaluationResult.armed_criteria_passed`; a run that completes naturally — armed or not — gates strict-AND via `all_criteria_passed`, so adding a block never changes the verdict of a run it didn't cut. Note the gate keys on the watcher having FIRED (`result.early_stop is not None`), not on confirmed truncation — an agent that ignores `should_stop`, or a stop firing on the final message, still gates armed-only. Every resolution-time guardrail violation is a hard error at resolution (plan *and* run); the one load-time case — a `stop_early:` block on a non-live criterion — is a pydantic schema error at task load, which the run surface reports as a skipped task like any other malformed task. A runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo` (incl. `gate_threshold` at stop time), report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. No blocks anywhere ⇒ behavior byte-for-byte unchanged. @@ -223,7 +232,7 @@ make plugin-reference # the plugin's bundled criteria reference from the models Editing `src/coder_eval/pricing.py` means editing `evalboard/lib/pricing.ts` too — it is a hand-copied mirror, and `evalboard/lib/__tests__/pricing-parity.test.ts` fails the build on drift in either direction. -Recent additions, each traceable to a shipped defect: **CE047** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE049** (never coalesce a possibly-unmeasured score to a numeric literal — `score or 0.0` publishes "measured and scored zero" while meaning "never measured", which is how an ungraded night reached four unfiltered `avg(Score)` dashboards as a real zero), **CE050** (no untyped `getattr` probe for a discriminated-union field — pyright cannot see the string, so a rename degrades the guard to a permanent no-op; scoped to criterion-shaped receivers because `command`/`tool`/`prompt` are far too common to flag on their own), **CE051** (a sandbox driver may not be rewritten silently — the driver IS the isolation boundary, so a downgrade must be an explicit, stamped, operator-visible decision), **CE052** (an `os._exit` must sit inside a branch testing `CODER_EVAL_IN_CONTAINER` — it is the right primitive only for reaping the container's own disposable main process, and `run_task_internal_command` armed its heartbeat watchdog, a daemon thread whose whole authority is `os._exit(137)`, unconditionally: a test that invoked the command in-process left the pytest worker holding that thread, which exited the worker 40s later inside an unrelated test file, naming a different test on each run and on each platform with no traceback — and the dead worker's lost coverage data then failed the gate as `65.13 < 80.00`, naming neither the test nor the cause), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's). +Recent additions, each traceable to a shipped defect: **CE047** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE049** (never coalesce a possibly-unmeasured score to a numeric literal — `score or 0.0` publishes "measured and scored zero" while meaning "never measured", which is how an ungraded night reached four unfiltered `avg(Score)` dashboards as a real zero), **CE050** (no untyped `getattr` probe for a discriminated-union field — pyright cannot see the string, so a rename degrades the guard to a permanent no-op; scoped to criterion-shaped receivers because `command`/`tool`/`prompt` are far too common to flag on their own), **CE051** (a sandbox driver may not be rewritten silently — the driver IS the isolation boundary, so a downgrade must be an explicit, stamped, operator-visible decision), **CE053** (no bare run-record filename literal outside `path_utils` — `TASK_JSON_FILENAME` shipped with a rename-safety rationale while twelve exact literals stayed unmigrated, including all three `rglob("task.json")` sites the constant's own comment cites as its reason to exist, so it created the second source of truth it argues against), **CE052** (an `os._exit` must sit inside a branch testing `CODER_EVAL_IN_CONTAINER` — it is the right primitive only for reaping the container's own disposable main process, and `run_task_internal_command` armed its heartbeat watchdog, a daemon thread whose whole authority is `os._exit(137)`, unconditionally: a test that invoked the command in-process left the pytest worker holding that thread, which exited the worker 40s later inside an unrelated test file, naming a different test on each run and on each platform with no traceback — and the dead worker's lost coverage data then failed the gate as `65.13 < 80.00`, naming neither the test nor the cause), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's). When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033–CE036 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE036 enforces the `live_verdict` determinism + monotonicity contract (`criteria/base.py`) that `EarlyStopWatcher`'s latching, deferred fail-stop, and flip-attribution silently depend on: monotonicity over arbitrary Python is undecidable, so instead of a static check it REPLAYS each live criterion against every prefix of recorded trajectories (`tests/lint/live_verdict_contract.py::CASES`) — on the authored ordering AND under seeded shuffles (`permuted_violations`, which catch order-sensitive bugs the authored walk misses) — and asserts the property directly, plus registry-derived coverage — every `LiveSuccessCriterion` in the union must have cases, and every polarity its instances claim via `live_decidable_polarities()` must actually be reached by one (otherwise a single always-`undecided` fixture would "cover" a type while proving nothing). Adding a live criterion therefore means adding `ContractCase`s in the same change. CE035 resolves every `steps..outputs.` / `needs..outputs.` reference in `.github/workflows/**` to a writer that actually produces that key — GitHub expands an unwritten output to the empty string, so a typo degrades a gate silently and actionlint models `steps.*.outputs` as an open string map. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.) diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index 35bc96d7..e85ef305 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -61,7 +61,7 @@ publishing different numbers for the same run. | Key | Type | Meaning | | --- | --- | --- | -| `pass_rate` | `float \| None` | `tasks_succeeded / tasks_graded` — errors are in the denominator, counted as misses; ungraded tasks are in neither. `None` on an empty or fully ungraded run (0/0 is unknown, not 0%). | +| `pass_rate` | `float \| None` | `tasks_succeeded / tasks_graded` — errors are in the denominator, counted as misses; ungraded tasks are in neither. `None` when the run is empty, **or when no row produced a verdict at all** — an `execute` night whose only non-ungraded rows are crashes was never measured, and reporting `0.0%` there reads as a total failure. | | `error_share` | `float \| None` | `tasks_error / tasks_graded`. Diagnostic only; never adjusts the rate. | | `tasks_graded` | `int` | `tasks_run - tasks_not_graded`. The denominator of both rates above. | | `total_cost_usd` | `float \| None` | **The bill**: agent + judge + simulator, summed over the rows. `None` when nothing could be priced. | @@ -248,10 +248,14 @@ The cross-variant summary: - `experiment_id`, `description`, `variant_ids`. - `task_summaries: list[TaskExperimentSummary]` — each `{task_id, variant_results, best_variant, is_tie, score_spread, replicate_count}`, - where each `VariantResult` carries `{variant_id, task_id, weighted_score` (`float | - None` — `null` on an ungraded or errored row)`, + where each `VariantResult` carries `{variant_id, task_id, weighted_score, final_status, duration_seconds, total_tokens, iteration_count, total_assistant_turns, reference_similarity, replicate_index, replicate_count}`. + `weighted_score` is `float | None`: `null` only when every replicate was + **ungraded**. An **errored** replicate counts as `0.0` — same rule as + `VariantAggregate.average_score` above. That is deliberate: dropping errored + rows would let a nightly where one image build failed report a *higher* + headline score than a clean one. - `variant_aggregates: dict[str, VariantAggregate]` — keyed by variant id. - `total_duration_seconds`. - `per_replicate_scores: dict[variant_id -> dict[task_id -> list[float]]]`. diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 2a72520d..24980077 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -42,6 +42,7 @@ coder-eval run tasks/hello_date.yaml --stream full # live LLM output | `--type, -T` | Override agent type for all tasks (`claude-code`, `codex`, `antigravity`, `opencode`, or a plugin kind). | | `--repeats` | Run each `(task, variant)` N times (≥1); overrides experiment/variant `repeats:`. See [Replicates](#replicates). | | `--resume` | Resume an interrupted run: skip tasks already finalized in `--run-dir` and run the rest, folding prior results into `run.json`. Requires `--run-dir`. See [Resuming a run](#resuming-a-run). | +| `--allow-host-grading` | `--resume` only. Grade an executed-but-ungraded `driver: docker` row on this host instead of refusing; the row is stamped `graded_on_host`. Rejected without `--resume`, since a fresh `run` grades inside the driver the task asks for. | | `--sample N` | For dataset-backed tasks, run a fixed-seed random N-row sample (reproducible; cheap smoke test). See [Bring Your Own Dataset](DATASETS.md). | | `--sample-per-stratum N` | For dataset-backed tasks, keep up to N rows per stratum (`stratify_field`). Overridden by `--sample`. Nondeterministic unless `dataset.sample_seed` is set — see [Bring Your Own Dataset](DATASETS.md). | | `--include-skipped` | Also run tasks marked `skip: true` in their YAML (off by default so CI keeps excluding them). | @@ -80,12 +81,21 @@ you want to iterate on afterwards. Grade the results later with budget breach still reports `ERROR` / `TIMEOUT` / `TOKEN_BUDGET_EXCEEDED` and still exits non-zero, exactly as under `run`. -Every `run` flag is available except two things, each refused rather than quietly +Exhausting `max_turns` is the one fact that does *not* become a status here. Under +`run` it decides the outcome only when the criteria fail — a max-turns trajectory +whose criteria pass is `SUCCESS` — so it is not knowable without grading. `execute` +records `max_turns_exhausted: true` on the row and finalizes `NOT_GRADED`; the later +grade reads the flag and reaches exactly the status `run` would have. Rows like this +are picked up by `run --resume`, which owes a grade to anything executed but never +scored — including a row that also timed out or tripped a budget. + +Every `run` flag is available except three things, each refused rather than quietly degraded: | Not supported | Why | | --- | --- | | `--junit-xml` | A JUnit report reports verdicts, and there are none. | +| `--allow-host-grading` | It decides how an executed-but-ungraded row is *graded*, and `execute` grades nothing. | | Simulation tasks | The dialog loop reads criteria results to decide whether to keep talking, so an ungraded dialog would silently change its own stopping behavior. Rejected by name at startup. | `stop_early:` blocks are also inert here: early stop exists to cut a run once the @@ -211,6 +221,8 @@ rather than as a verdict. Override either default with `--in-place` / `--copy`. | `--in-place / --copy` | Grade where the files are, or copy first. Default: in-place for a run directory, copy for a plain work directory. | | `--preserve / --no-preserve` | Preserve sandbox after evaluation (default: preserve). Ignored when grading in place — an adopted directory is never moved or deleted. | | `--run-dir` | Where the graded `task.json` lands (default: auto-generated timestamped dir in `runs/`). | +| `--allow-recorded-commands` | Accept a rebuilt config that would run shell (`run_command` criteria, judges, `pre_run`/`post_run`) or install packages on this host. Refused by default — a run directory is a shareable artifact, so its recorded config is untrusted input. | +| `--allow-host-grading` | Grade a `driver: docker` task on this host instead of refusing. The row is stamped `graded_on_host` so it is never silently compared with a container-graded one. | | `--verbose, -v` | DEBUG-level logging | A re-grade refuses to run if the task's `reference:` directory changed since the diff --git a/docs/tutorials/02-ci-pipeline.md b/docs/tutorials/02-ci-pipeline.md index f6cb76c9..2c8c6839 100644 --- a/docs/tutorials/02-ci-pipeline.md +++ b/docs/tutorials/02-ci-pipeline.md @@ -119,9 +119,14 @@ jobs: for p in sorted(pathlib.Path(sys.argv[1]).rglob("task.json"))] for r in rows: print(f"- {r.get('task_id','?')}: {r.get('final_status','?')}") - ok = sum(1 for r in rows if r.get("final_status") == "SUCCESS") - print(f"\n**{ok}/{len(rows)} PASS**") - sys.exit(0 if rows and ok == len(rows) else 1) + # NOT_GRADED is a fourth category, not a failure: `coder-eval execute` + # runs the agent and deliberately scores nothing. Counting such a row + # as a miss fails a pipeline that has not measured anything yet. + graded = [r for r in rows if r.get("final_status") != "NOT_GRADED"] + ok = sum(1 for r in graded if r.get("final_status") == "SUCCESS") + print(f"\n**{ok}/{len(graded)} PASS**" + + (f" ({len(rows) - len(graded)} not graded)" if len(graded) != len(rows) else "")) + sys.exit(0 if graded and ok == len(graded) else 1) PY - name: Upload run reports diff --git a/evalboard/app/trends/trends-view.tsx b/evalboard/app/trends/trends-view.tsx index 611b7025..b1e7eb4a 100644 --- a/evalboard/app/trends/trends-view.tsx +++ b/evalboard/app/trends/trends-view.tsx @@ -32,6 +32,7 @@ import { VersionChip } from "@/app/_components/version-list"; import { HarnessSelector } from "@/app/_components/harness-selector"; import { harnessShortLabel } from "@/app/_components/harness-badge"; import { fetchTaskHistoryAction } from "./actions"; +import { isGraded, isPassStatus } from "@/lib/status"; function fmtUsd(c: number | null): string { if (c == null) return "—"; @@ -146,9 +147,13 @@ function sortTasks( function statusFill(status: string | null): string { if (status == null) return "bg-gray-300"; - if (status === "SUCCESS") return "bg-green-500"; - // All non-success outcomes share one red — the trends view treats every - // failure mode as equally bad rather than ranking FAILED vs ERROR. + if (isPassStatus(status)) return "bg-green-500"; + // An ungraded row was never measured, so it is not a failure. Painting it + // red contradicted lib/trends.ts, which had just excluded those very rows + // from the pass rate feeding this same timeline. + if (!isGraded(status)) return "bg-gray-300"; + // All non-success graded outcomes share one red — the trends view treats + // every failure mode as equally bad rather than ranking FAILED vs ERROR. return "bg-red-500"; } diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts index 3f4f4f7b..92774063 100644 --- a/evalboard/lib/__tests__/overview.test.ts +++ b/evalboard/lib/__tests__/overview.test.ts @@ -1258,3 +1258,82 @@ describe("adhocRunDate", () => { ]); }); }); + +describe("ungraded rows leave both sides of every rate", () => { + // The three denominators this PR changed had no assertions at all. Each + // case below is the arithmetic invariant, not the code shape: a row that + // was never measured must leave the numerator AND the denominator, or a + // `coder-eval execute` night publishes a measured-looking 0%. + function ungradedPerRun(id: string, tasks: RunOverviewTask[]): PerRun { + return { + id, + overview: { + id, + tasks, + totalCostUsd: null, + taskDurationSeconds: null, + componentShas: [], + }, + reviewTagCounts: {}, + reviewTagsByTask: {}, + adhoc: false, + title: null, + }; + } + + test("buildTagTaskRows: executed = appearances - matureSkips - ungraded", () => { + const TAG = "path-to-ga"; + const rows = buildTagTaskRows( + [ + ungradedPerRun("r1", [ + task({ taskId: "a", tags: [TAG], status: "SUCCESS" }), + ]), + ungradedPerRun("r2", [ + task({ taskId: "a", tags: [TAG], status: "NOT_GRADED" }), + ]), + ], + TAG, + ); + + expect(rows[0].appearances).toBe(2); + expect(rows[0].ungraded).toBe(1); + expect(rows[0].executed).toBe(1); + expect(rows[0].passRate).toBe(100); + }); + + test("buildTagTaskRows: an all-ungraded task has no pass rate at all", () => { + const TAG = "path-to-ga"; + const rows = buildTagTaskRows( + [ + ungradedPerRun("r1", [ + task({ taskId: "a", tags: [TAG], status: "NOT_GRADED" }), + ]), + ], + TAG, + ); + + expect(rows[0].executed).toBe(0); + expect(rows[0].passRate).toBeNull(); + }); + + test("turnBudgetRateForTasks: an ungraded budgeted row is not a budget miss", () => { + // The confirmed miss. `status` is a free string, so neither tsc nor + // assertNever could see that a raw `!== "SUCCESS"` books the fourth + // category as a failure — it entered `eligible` and never + // `withinBudget`, publishing 0%. + const withUngraded = turnBudgetRateForTasks([ + task({ status: "SUCCESS", expectedTurns: 10, visibleTurns: 5 }), + task({ status: "NOT_GRADED", expectedTurns: 10, visibleTurns: 99 }), + ]); + + expect(withUngraded).toBe(100); + }); + + test("turnBudgetRateForTasks: an all-ungraded scope reports null, not 0%", () => { + expect( + turnBudgetRateForTasks([ + task({ status: "NOT_GRADED", expectedTurns: 10, visibleTurns: 4 }), + ]), + ).toBeNull(); + }); +}); diff --git a/evalboard/lib/__tests__/trends.test.ts b/evalboard/lib/__tests__/trends.test.ts index 1a039d95..79e310a9 100644 --- a/evalboard/lib/__tests__/trends.test.ts +++ b/evalboard/lib/__tests__/trends.test.ts @@ -225,3 +225,30 @@ describe("historyForTaskInner", () => { expect(byRun.r2).toBe(false); }); }); + +describe("aggregate — ungraded rows", () => { + // The parallel of the "mature skip" block above, for the fourth status + // category. The denominator changed here with nothing asserting it: an + // ungraded row must leave BOTH sides, or a `coder-eval execute` run drags + // every task's trend down as though it had failed. + test("leave both sides of the pass rate", () => { + const { trends } = aggregate([ + perRun("r1", [task({ status: "SUCCESS" })]), + perRun("r2", [task({ status: "NOT_GRADED" })]), + ]); + + expect(trends[0].totalRuns).toBe(1); + expect(trends[0].successRuns).toBe(1); + expect(trends[0].passRate).toBe(1); + }); + + test("a fully ungraded task reports no runs rather than a 0% pass rate", () => { + const { trends } = aggregate([ + perRun("r1", [task({ status: "NOT_GRADED" })]), + perRun("r2", [task({ status: "NOT_GRADED" })]), + ]); + + expect(trends[0].totalRuns).toBe(0); + expect(trends[0].successRuns).toBe(0); + }); +}); diff --git a/evalboard/lib/__tests__/watchlist.test.ts b/evalboard/lib/__tests__/watchlist.test.ts index c1614d39..5355f023 100644 --- a/evalboard/lib/__tests__/watchlist.test.ts +++ b/evalboard/lib/__tests__/watchlist.test.ts @@ -299,3 +299,35 @@ describe("empty window", () => { expect(data.windowSize).toBe(0); }); }); + +describe("ungraded rows", () => { + // The denominators in leaderboard() and attention() gained an `isGraded` + // filter with nothing asserting it. An ungraded row must leave BOTH sides: + // counted only in the denominator it reads as a failure, which is exactly + // what would put a `coder-eval execute` night at the top of the watchlist. + test("leave both sides of a skill's pass rate", () => { + const data = buildWatchlist([ + perRun("2026-01-02", [ + task({ taskId: "a", skill: "alpha", status: "SUCCESS" }), + task({ taskId: "b", skill: "alpha", status: "NOT_GRADED" }), + ]), + ]); + + expect(data.leaderboard).toEqual([ + { skill: "alpha", passRate: 1, outcomes: 1 }, + ]); + }); + + test("an all-ungraded skill raises no attention at all", () => { + const runs = Array.from({ length: 4 }, (_, i) => + perRun(`2026-01-0${4 - i}`, [ + task({ taskId: "a", skill: "unmeasured", status: "NOT_GRADED" }), + ]), + ); + + const { topAttention, leaderboard } = buildWatchlist(runs); + + expect(topAttention).toHaveLength(0); + expect(leaderboard).toEqual([]); + }); +}); diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index e8e26f51..c28c7704 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -81,7 +81,14 @@ export function turnBudgetRateForTasks(tasks: RunOverviewTask[]): number | null for (const t of tasks) { const hasBudget = t.expectedTurns != null && t.expectedTurns >= 1; if (!hasBudget) continue; // unbudgeted tasks never count, success or fail - if (t.status !== "SUCCESS") { + // An ungraded row leaves BOTH sides, like every other rate in this app. + // A raw `!== "SUCCESS"` here put it in `eligible` and never in + // `withinBudget`, i.e. booked it as a budget MISS — so one + // `coder-eval execute` run published a measured-looking Turn Budget of + // 0%. `status` is a free string, so neither tsc nor assertNever could + // see it; the typed helper is the only thing that can. + if (!isGraded(t.status)) continue; + if (!isPassStatus(t.status)) { // A budgeted task that didn't succeed never stayed within budget. eligible += 1; continue; @@ -111,7 +118,10 @@ export function withinExpectedTimeRateForTasks( let eligible = 0; let within = 0; for (const t of tasks) { - if (t.status !== "SUCCESS" || t.matureSkipped) continue; + // Behaviourally identical to the old raw literal (a non-pass already + // left both sides here), converted so this file has ONE rule for what + // "passed" means rather than two spellings of it. + if (!isPassStatus(t.status) || t.matureSkipped) continue; const verdict = withinExpectedTime(t.durationSeconds, t.expectedSeconds); if (verdict === null) continue; // unscored, or no duration → can't judge eligible += 1; diff --git a/evalboard/lib/status.ts b/evalboard/lib/status.ts index 7e257479..c79b9519 100644 --- a/evalboard/lib/status.ts +++ b/evalboard/lib/status.ts @@ -94,10 +94,25 @@ export function assertNever(x: never): never { throw new Error(`Unhandled status category: ${String(x)}`); } -// Default table sort: failures and errors first, unknowns next, passes last. +// Default table sort: failures and errors first, ungraded/unknown next, passes +// last. A `switch` with `assertNever`, not the if/else-with-catch-all this +// file's own header calls out: the catch-all sorted the new "ungraded" bucket +// into the "unknown" rank by fall-through rather than by decision, and a sixth +// category would land there just as silently. export function statusSortRank(status: string | null): number { const c = statusCategory(status); - if (c === "failed" || c === "error") return 0; - if (c === "passed") return 2; - return 1; + switch (c) { + case "failed": + case "error": + return 0; + case "passed": + return 2; + case "ungraded": + case "unknown": + // Between the two: an ungraded row is not a failure to rank first, + // and not a pass to bury last. + return 1; + default: + return assertNever(c); + } } diff --git a/evalboard/lib/trends.ts b/evalboard/lib/trends.ts index 2480fffc..2145f735 100644 --- a/evalboard/lib/trends.ts +++ b/evalboard/lib/trends.ts @@ -12,7 +12,7 @@ import { } from "./overview"; import { DEFAULT_HARNESS } from "./harness"; import { DEFAULT_SOURCE, type Source } from "./sources"; -import { isGraded } from "./status"; +import { isGraded, isPassStatus } from "./status"; import { taskCarriesRepoTag } from "./tags"; import type { ComponentSha } from "./runs"; @@ -162,7 +162,7 @@ export function aggregate(perRun: PerRun[]): TrendsData { status: t.status, matureSkipped: t.matureSkipped ?? false, }); - if (t.status === "SUCCESS") { + if (isPassStatus(t.status)) { b.successCount += 1; // A mature task the nightly skipped still counts as a pass, but // it wasn't executed — its row carries 0 cost / 0 duration and diff --git a/evalboard/lib/watchlist.ts b/evalboard/lib/watchlist.ts index 79da9ebf..f231ba12 100644 --- a/evalboard/lib/watchlist.ts +++ b/evalboard/lib/watchlist.ts @@ -11,7 +11,7 @@ import type { PerRun } from "./overview"; import type { RunOverviewTask } from "./runs"; -import { isGraded } from "./status"; +import { isGraded, isPassStatus } from "./status"; import { timeRatio } from "./timing"; import { turnRatio } from "./turns"; @@ -99,7 +99,10 @@ function runsNewestFirst(perRun: PerRun[]): LoadedRun[] { .map((r) => ({ id: r.id, tasks: r.overview!.tasks })); } -const isPass = (status: string | null) => status === "SUCCESS"; +// Delegates rather than restating `=== "SUCCESS"`: this file already had to +// learn about the ungraded bucket, and a second private spelling of "passed" +// is how one surface ends up disagreeing with the rest. +const isPass = (status: string | null) => isPassStatus(status); const clamp01 = (n: number) => (n < 0 ? 0 : n > 1 ? 1 : n); const mean = (xs: number[]) => @@ -217,6 +220,14 @@ export function attention(runs: LoadedRun[]): AttentionRow[] { } } if (appeared < floor) continue; + // Nothing measured ⇒ nothing to say. Without this, a skill whose rows + // were all ungraded fell into `outcomes ? … : 0` below and scored + // passRate 0 / failRate 1 — the maximum FAIL_WEIGHT — so a + // `coder-eval execute` night would put its most-run skill at the TOP of + // an exec-triage hero, described as chronically failing. The same + // "never measured" vs "measured and scored zero" confusion CE049 exists + // to prevent on the Python side. + if (outcomes === 0) continue; const passRate = outcomes ? passes / outcomes : 0; const failRate = 1 - passRate; diff --git a/pyproject.toml b/pyproject.toml index b8978938..da637243 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -220,16 +220,42 @@ external = [ "CE011", "CE012", "CE013", + "CE014", + "CE015", + "CE016", + "CE017", "CE018", + "CE019", + "CE020", + "CE021", + "CE022", + "CE023", + "CE024", + "CE025", + "CE026", + "CE027", + "CE028", + "CE029", + "CE030", + "CE031", + "CE032", "CE033", + "CE034", "CE035", + "CE036", "CE037", "CE038", "CE039", + "CE043", + "CE045", + "CE046", + "CE047", + "CE048", "CE049", "CE050", "CE051", "CE052", + "CE053", ] # custom architectural lint rules (tests/lint/) [tool.ruff.lint.pylint] diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index bcf10966..d2603f7f 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -28,6 +28,7 @@ load_prior_result, regrade_in_place, restore_pre_grade_record, + stamp_host_grading, task_from_prior, verify_reference_unchanged, ) @@ -49,6 +50,24 @@ logger = logging.getLogger(__name__) +def resolve_grade_in_place(target: EvaluateTarget, in_place: bool | None) -> bool: + """Whether this grade runs in the target directory or in a copy of it. + + In-place is the default for a run directory: that workspace is the run's own + output, and copying it filters build artifacts (``node_modules``, ``dist``, + ``.venv``) out of the grade, so a criterion reading them fails as a copying + artifact rather than as a verdict. A plain work directory defaults to + copying, because criteria can mutate the target and it is the user's own + tree. + + A function rather than an expression because two places need the answer and + one of them — the recorded-shell refusal — changes what the command is + willing to execute. A restated copy of the rule silently stops matching the + moment the default moves. + """ + return in_place if in_place is not None else (target.mode is EvaluateMode.RUN_DIR) + + @dataclass(frozen=True) class _ResolvedInputs: """Everything the two positionals + ``--workspace`` decide, resolved once.""" @@ -127,17 +146,22 @@ def _resolve_run_dir_or_work_dir( task, source_yaml = load_task(target.task_file) console.print(f"[dim]Grading with {target.task_file} (overrides the run's recorded config).[/dim]") else: - # pre_run/post_run are skipped on the in-place path (an adopted - # workspace must not have its hooks re-run over the agent's - # deliverables), so there they are not shell the run dir can make - # this host execute — only --copy re-runs them. Mirrors the - # grade_in_place default computed in run_evaluation. - hooks_will_run = not (in_place if in_place is not None else True) + # pre_run/post_run and the sandbox's installers both run only on the + # --copy path (an adopted workspace must not have its hooks re-run + # over the agent's deliverables, and `adopt` installs nothing), so + # in place they are not capabilities the run dir can reach. + # + # Derived through the SAME function `run_evaluation` uses, not + # restated. This value decides whether recorded shell is refused, so + # a second copy of the rule would keep answering the old question if + # the default ever moved — and silently stop covering commands that + # then do run. + setup_will_run = not resolve_grade_in_place(target, in_place) task, source_yaml = task_from_prior( prior, target.target, allow_recorded_commands=allow_recorded_commands, - include_hooks=hooks_will_run, + include_setup_phase=setup_will_run, ) work_dir = workspace or default_workspace(target.target, prior) recorded_source = prior.task_config.source_file if prior.task_config else None @@ -341,11 +365,7 @@ def run_evaluation( prior = inputs.prior target = inputs.target - # In-place is the default for a run directory: that workspace is the run's - # own output and copying it would filter build artifacts out of the grade. - # A plain work directory defaults to copying, because criteria can mutate the - # target and it is the user's own tree. - grade_in_place = in_place if in_place is not None else (target.mode is EvaluateMode.RUN_DIR) + grade_in_place = resolve_grade_in_place(target, in_place) try: prepared_run_dir = prepare_run_directory(run_dir) @@ -400,12 +420,39 @@ async def _setup_and_run() -> EvaluationResult: task_file=task_file, sandbox=sandbox, variant_id=prior.variant_id if prior is not None else "evaluate", + replicate_index=_replicate_index_of(target.target), source_yaml=source_yaml, prior_result=prior, ) - return await orchestrator.run() + graded = await orchestrator.run() + # Same stamp the delegating branch gets from `regrade_in_place`. Line 357 + # above accepted the docker→host downgrade for THIS branch too, and + # CLAUDE.md, the user guide and CE051's own noqa all state the stamp as + # unconditional — so `evaluate --copy --allow-host-grading` + # was writing an unstamped host verdict that nothing downstream could + # tell apart from a container-graded one. + stamp_host_grading(graded, task) + return graded result = asyncio.run(_setup_and_run()) + _report_and_exit(result, task=task, prior=prior, target=target, prepared_run_dir=prepared_run_dir) + + +def _report_and_exit( + result: EvaluationResult, + *, + task: TaskDefinition, + prior: EvaluationResult | None, + target: EvaluateTarget, + prepared_run_dir: Path, +) -> None: + """Render the graded row, write it back, and choose the exit code. + + Split out of ``run_evaluation`` because it answers a different question — + what to TELL the operator about a result that already exists — and because + the two together grew past the function-size bound the moment the inherited + status handling landed. Always raises ``typer.Exit``. + """ # BEFORE the count guard below. A grading crash returns a populated ERROR # result with an EMPTY criteria list (Orchestrator.run() converts internal @@ -413,7 +460,20 @@ async def _setup_and_run() -> EvaluationResult: # first and the user is told only "Result count mismatch: got 0, expected 2" # — the real error is never printed, and the "still re-gradeable" notice is # unreachable on exactly the path it was written for. - if result.final_status is FinalStatus.ERROR: + # Whether the terminal status describes THIS pass or was carried over from + # the run being graded. `Orchestrator._terminal_status` preserves a prior + # execution fact (TIMEOUT / ERROR / BUILD_FAILED / a budget stop) because + # grading may not overturn it — so reading `result.final_status` as this + # pass's own outcome misreports both arms below. It made a preserved ERROR + # print the ORIGINAL run's crash message as though grading had crashed, + # claim the row was "left ungraded" (it was not — the restored record still + # reads ERROR), and throw away a verdict that had just been computed at + # 1.000; and it made a preserved TIMEOUT exit 0 under "All criteria passed", + # so a CI wrapper reading the exit code goes green on a row run.json counts + # as failed. + inherited = prior is not None and prior.final_status.is_execution_fact + + if result.final_status is FinalStatus.ERROR and not inherited: console.print(f"\n[red]✗ Evaluation error: {result.error_message}[/red]") if prior is not None: # A grading-time crash (a failing checker, an unreachable judge) is @@ -478,6 +538,16 @@ async def _setup_and_run() -> EvaluationResult: ) _write_back(target.target, result) + if result.final_status.is_execution_fact: + # The criteria tally is real and worth printing — it is why the table + # above still renders — but it is not the row's outcome. run.json will + # count this row under its preserved status, and the exit code must + # agree with run.json rather than with the tally. + console.print( + f"\n[red]Criteria: {passed}/{total} passed, but the run itself ended as " + + f"{result.final_status.value} — grading cannot overturn that.[/red]" + ) + raise typer.Exit(1) if failed == 0: console.print("\n[green]All criteria passed! ✓[/green]") raise typer.Exit(0) diff --git a/src/coder_eval/cli/report_command.py b/src/coder_eval/cli/report_command.py index 6c5fcfc4..a64d0cf7 100644 --- a/src/coder_eval/cli/report_command.py +++ b/src/coder_eval/cli/report_command.py @@ -6,6 +6,7 @@ from rich.markdown import Markdown from ..models import EvaluationResult +from ..path_utils import TASK_JSON_FILENAME from ..reports import ReportGenerator from ..reports_html import write_task_html from .console import console @@ -96,7 +97,7 @@ def _regenerate_html_reports(run_dir: Path, output_file: Path | None) -> None: report is written to that file. Otherwise each task.html is written next to its task.json. """ - task_json_paths = sorted(run_dir.rglob("task.json")) + task_json_paths = sorted(run_dir.rglob(TASK_JSON_FILENAME)) if not task_json_paths: console.print(f"[red]Error: no task.json files found under {run_dir}[/red]") raise typer.Exit(1) diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index 336370cd..d3d718e8 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -444,6 +444,16 @@ def run_pipeline( # --resume needs an explicit run dir to resume into (auto-generated dirs are always fresh). if resume and run_dir is None: raise typer.BadParameter("--resume requires --run-dir pointing at the run to continue.") + # --allow-host-grading only reaches anything from inside the `if resume:` + # branch below, so without --resume it parsed, was accepted, and did nothing + # at all — no warning, no error. The sibling mode-scoped flag on the same + # feature (`evaluate --workspace`) hard-errors on exactly this misuse; two + # new flags behaving differently for one user mistake is the inconsistency. + if allow_host_grading and not resume: + raise typer.BadParameter( + "--allow-host-grading applies to --resume only (it decides how an executed-but-ungraded " + + "row is graded). A fresh `run` grades inside the driver the task asks for." + ) # Parse tag filters include_tags = {t.strip() for t in tags.split(",") if t.strip()} if tags else None diff --git a/src/coder_eval/cli/run_task_internal_command.py b/src/coder_eval/cli/run_task_internal_command.py index bcee7c28..ac3c2bb2 100644 --- a/src/coder_eval/cli/run_task_internal_command.py +++ b/src/coder_eval/cli/run_task_internal_command.py @@ -34,6 +34,7 @@ CONTAINER_TASK_DIR, ConfigLineageEntry, PreservationMode, + SandboxConfig, ) from coder_eval.orchestration.task_loader import load_task @@ -51,37 +52,13 @@ def heartbeat_is_alive(current: str, last_counter: str, current_mtime: float, la return bool(current and current != last_counter) or current_mtime > last_mtime -def run_task_internal_command( - input_dir: Path = typer.Option( # noqa: B008 - Path(CONTAINER_INPUT_DIR), - "--input", - help="Directory containing task.yaml and context.json (bind-mounted by host).", - ), - output_dir: Path = typer.Option( # noqa: B008 - Path(CONTAINER_OUTPUT_DIR), - "--output", - help="Directory to write task.json/task.html into (bind-mounted by host).", - ), - task_dir: Path = typer.Option( # noqa: B008 - Path(CONTAINER_TASK_DIR), - "--task-dir", - help="Original task directory mount (used to resolve relative template paths).", - ), - verbose: bool = typer.Option( - False, - "--verbose", - "-v", - help="Enable verbose (DEBUG level) logging", - ), -) -> None: - """Run a single staged task inside the container.""" - # Use the same logging path as the host CLI so LOG_LEVEL from the - # forwarded env is honoured. Without this, root stays at INFO and the - # DEBUG-level task_log_handler attached by Orchestrator never sees the - # agent's per-tool-call DEBUG records. - log_level = "DEBUG" if verbose else settings.log_level - setup_logging(level=log_level) +def _arm_host_heartbeat_watchdog(output_dir: Path) -> None: + """Start the orphan-container reaper, but only inside the container. + A module-level function rather than an inline block so the only + process-lethal code in this command sits behind one named, testable seam + instead of being a side effect of the command body. + """ # Start the host-heartbeat watchdog: if the host process dies # ungracefully (SIGKILL, Claude-Code Escape, crash) before it can # `docker kill` us, the heartbeat file in output_dir goes stale and @@ -159,6 +136,40 @@ def _watch_host_heartbeat() -> None: else: logger.debug("Not in a container; host-heartbeat watchdog not armed.") + +def run_task_internal_command( + input_dir: Path = typer.Option( # noqa: B008 + Path(CONTAINER_INPUT_DIR), + "--input", + help="Directory containing task.yaml and context.json (bind-mounted by host).", + ), + output_dir: Path = typer.Option( # noqa: B008 + Path(CONTAINER_OUTPUT_DIR), + "--output", + help="Directory to write task.json/task.html into (bind-mounted by host).", + ), + task_dir: Path = typer.Option( # noqa: B008 + Path(CONTAINER_TASK_DIR), + "--task-dir", + help="Original task directory mount (used to resolve relative template paths).", + ), + verbose: bool = typer.Option( + False, + "--verbose", + "-v", + help="Enable verbose (DEBUG level) logging", + ), +) -> None: + """Run a single staged task inside the container.""" + # Use the same logging path as the host CLI so LOG_LEVEL from the + # forwarded env is honoured. Without this, root stays at INFO and the + # DEBUG-level task_log_handler attached by Orchestrator never sees the + # agent's per-tool-call DEBUG records. + log_level = "DEBUG" if verbose else settings.log_level + setup_logging(level=log_level) + + _arm_host_heartbeat_watchdog(output_dir) + task_yaml = input_dir / "task.yaml" context_json = input_dir / "context.json" if not task_yaml.exists(): @@ -169,8 +180,20 @@ def _watch_host_heartbeat() -> None: raise typer.Exit(2) context = json.loads(context_json.read_text(encoding="utf-8")) - variant_id: str = context["variant_id"] - replicate_index: int = context.get("replicate_index", 0) + # Checked, not just annotated. `json.loads` returns `Any`, so pyright accepts + # `variant_id: str = context["variant_id"]` for a value that may be anything + # at all — the annotation reads like a guarantee and enforces nothing. A + # `"replicate_index": "00"` then reached `build_task_run_dir` typed as `int`. + # This is the host→container boundary; the comment below (about `grade` + # being the one raw value) was only true because these two looked checked. + variant_id = context["variant_id"] + if not isinstance(variant_id, str): + typer.echo(f"FATAL: context.json 'variant_id' must be a string, got {variant_id!r}", err=True) + raise typer.Exit(2) + replicate_index = context.get("replicate_index", 0) + if not isinstance(replicate_index, int) or isinstance(replicate_index, bool): + typer.echo(f"FATAL: context.json 'replicate_index' must be an integer, got {replicate_index!r}", err=True) + raise typer.Exit(2) # The host resolves the driver-derived default before dispatch; the container # obeys it verbatim. This command only ever runs inside the docker driver, so # a missing key falls back to the docker default (DIRECT_WRITE) — a deliberate @@ -179,10 +202,10 @@ def _watch_host_heartbeat() -> None: # `coder-eval run` vs `coder-eval execute`, decided host-side. Defaults to # True (grade) so a host that predates `execute` — which never writes the # key — keeps its exact behavior. - # Coerced, not annotated: every other value crossing this boundary goes - # through a validating constructor, but `grade` was taken raw — so a - # hand-edited or older-format `"grade": "false"` arrives as a truthy str - # typed as bool and silently grades a run that asked not to be graded. + # Coerced, not annotated, like every other value crossing this boundary — + # `grade` was once the only raw one, so a hand-edited or older-format + # `"grade": "false"` arrived as a truthy str typed as bool and silently + # graded a run that asked not to be graded. grade_raw = context.get("grade", True) if not isinstance(grade_raw, bool): typer.echo(f"FATAL: context.json 'grade' must be a boolean, got {grade_raw!r}", err=True) @@ -218,7 +241,14 @@ def _watch_host_heartbeat() -> None: # container the docker driver asked for, so the isolation the driver # names is present, not bypassed; a nested docker would be both wrong # and impossible (no docker CLI in the image). - task = task.model_copy(update={"sandbox": task.sandbox.model_copy(update={"driver": "tempdir"})}) # noqa: CE051 + # Re-validated rather than `model_copy(update=...)`, matching its sibling + # `regrade.grading_sandbox_config`: `update` skips BOTH pydantic and + # pyright, so a typo would produce a SandboxConfig violating its own + # `Literal` and only surface far downstream. Two driver-rewrite sites + # landing in one change with two different levels of type safety is how + # the weaker one becomes the pattern people copy. + rewritten = SandboxConfig.model_validate({**task.sandbox.model_dump(), "driver": "tempdir"}) # noqa: CE051 + task = task.model_copy(update={"sandbox": rewritten}) output_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/coder_eval/isolation/docker_runner.py b/src/coder_eval/isolation/docker_runner.py index 3b46631b..57bac3da 100644 --- a/src/coder_eval/isolation/docker_runner.py +++ b/src/coder_eval/isolation/docker_runner.py @@ -40,7 +40,13 @@ ResourceLimits, ) from coder_eval.orchestration.evaluation import resolve_host_reference_dir -from coder_eval.path_utils import REFERENCE_COPY_IGNORE, ignore_patterns_and_symlinks, rmtree_restrictive +from coder_eval.path_utils import ( + REFERENCE_COPY_IGNORE, + TASK_JSON_FILENAME, + ignore_patterns_and_symlinks, + rmtree_restrictive, + write_text_atomic, +) from coder_eval.streaming.callbacks import safe_emit from coder_eval.streaming.wire import deserialize_event, has_prefix from coder_eval.utils import get_default_docker_image_tag @@ -845,7 +851,7 @@ async def _parse_result_or_raise(self, output_dir: Path, returncode: int, log_pa task.json and raise ``DockerRunError`` so the batch dispatcher records the failure as an ERROR-status result. """ - task_json = output_dir / "task.json" + task_json = output_dir / TASK_JSON_FILENAME if not await asyncio.to_thread(task_json.exists): # The container died before its orchestrator's `finally` could # write task.json (e.g. it was torn down by the cleanup above @@ -871,10 +877,10 @@ async def _parse_result_or_raise(self, output_dir: Path, returncode: int, log_pa # rather than crashing with an uncaught ValidationError/JSONDecodeError. raise await self._handle_malformed_task_json(task_json, log_path, exc) from exc self._warn_on_version_mismatch(result) - self._assert_grade_honored(result) + self._assert_grade_honored(result, task_json) return result - def _assert_grade_honored(self, result: EvaluationResult) -> None: + def _assert_grade_honored(self, result: EvaluationResult, task_json: Path | None = None) -> None: """Fail loudly when `execute` came back with a graded verdict. ``grade`` crosses the boundary only through ``context.json``. An image @@ -883,16 +889,44 @@ def _assert_grade_honored(self, result: EvaluationResult) -> None: against a stale image would silently produce SUCCESS/FAILURE rows that look like a normal graded run. Version skew must not change what a command MEANS, so refuse the row rather than publish it. + + ``task_json`` is the on-disk record, quarantined before the raise. The + refusal used to be in-memory only, which left the graded ``task.json`` + sitting in the bind-mounted host run dir: a later + ``execute --resume`` read it back as a completed row (its category is + ``succeeded``, so the resume partition files it under prior results) and + plain ``aggregate`` folded it straight into ``run.json`` — publishing + exactly the row this guard declined to publish. Refusing in memory while + leaving contradictory bytes on disk is not a refusal. """ - if self.grade or result.final_status.is_execution_fact: + if self.grade: return - if result.final_status is not FinalStatus.NOT_GRADED: - raise DockerRunError( - "`coder-eval execute` asked the container not to grade, but it returned " - + f"{result.final_status.value} with {len(result.success_criteria_results)} criterion " - + "result(s). The runtime image predates `execute` and ignored the request; " - + "rebuild or pull a matching agent image." - ) + # Keyed on EVIDENCE, not on the label. Exempting every execution-fact + # status let a stale image return a fully graded MAX_TURNS_EXHAUSTED row + # — criteria vector, weighted score and all — unchallenged, because the + # exemption exists for statuses a *fresh* image also produces, and a + # fresh one produces them with neither. The question is not "what status + # is this" but "did it grade". + graded_anyway = bool(result.success_criteria_results) or result.weighted_score is not None + if not graded_anyway and ( + result.final_status.is_execution_fact or result.final_status is FinalStatus.NOT_GRADED + ): + return + if task_json is not None: + # Same sidecar pattern as `_handle_malformed_task_json`. Best-effort: + # a failed move is logged, never masking the raise below. + sidecar = task_json.with_suffix(task_json.suffix + ".graded") + try: + os.replace(task_json, sidecar) + logger.warning("Quarantined the refused graded record to %s", sidecar) + except OSError as exc: + logger.warning("Could not quarantine %s: %s", task_json, exc) + raise DockerRunError( + "`coder-eval execute` asked the container not to grade, but it returned " + + f"{result.final_status.value} with {len(result.success_criteria_results)} criterion " + + "result(s). The runtime image predates `execute` and ignored the request; " + + "rebuild or pull a matching agent image." + ) async def _handle_malformed_task_json(self, task_json: Path, log_path: Path, exc: ValueError) -> DockerRunError: """Degrade a present-but-malformed task.json; return the DockerRunError to raise. @@ -939,7 +973,9 @@ async def _record_build_failure(self, exc: DockerBuildError) -> None: await asyncio.to_thread(self.rt.run_dir.mkdir, parents=True, exist_ok=True) log_path = self.rt.run_dir / "docker.log" await asyncio.to_thread(log_path.write_text, exc.build_log or str(exc), encoding="utf-8") - await self._write_synthetic_task_json(self.rt.run_dir / "task.json", exc, status=FinalStatus.BUILD_FAILED) + await self._write_synthetic_task_json( + self.rt.run_dir / TASK_JSON_FILENAME, exc, status=FinalStatus.BUILD_FAILED + ) except OSError as io_exc: # pragma: no cover - defensive logger.warning("Failed to record build failure for %s: %s", self.rt.task.task_id, io_exc) @@ -968,9 +1004,15 @@ async def _write_synthetic_task_json( def _write() -> None: if target.exists(): return - tmp = target.with_suffix(target.suffix + ".synthetic.tmp") - tmp.write_text(result.model_dump_json(indent=2), encoding="utf-8") - os.replace(tmp, target) + # Through `write_text_atomic` like every other writer of this file. + # The hand-rolled tmp+replace here used `Path.write_text`, which + # FOLLOWS symlinks — so a pre-planted `task.json.synthetic.tmp` in a + # run directory (a shareable artifact, bind-mounted writable into the + # agent's own container) redirected this harness-privileged write to + # any path the grading user could reach. It also falsified the + # helper's "one writer, so the crash semantics cannot differ" claim, + # which is the property future readers rely on. + write_text_atomic(target, result.model_dump_json(indent=2)) try: await asyncio.to_thread(_write) diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py index d6b8745a..b3b3671a 100644 --- a/src/coder_eval/models/experiment.py +++ b/src/coder_eval/models/experiment.py @@ -217,8 +217,9 @@ class VariantAggregate(BaseModel): # noqa: CE009 -- persisted result model; rou """Aggregated statistics for a single variant across all tasks. ``pass_rate`` uses the same denominator as ``RunSummary.pass_rate``: every task - the variant ran, errors included as misses. Otherwise an A/B whose variants - error at different rates compares two different denominators. + the variant GRADED, errors included as misses, ungraded tasks leaving BOTH + sides. Otherwise an A/B whose variants error at different rates compares two + different denominators. """ variant_id: str diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 43b1278f..03feabcd 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -1156,7 +1156,7 @@ def _check_task_count_invariant(self) -> RunSummary: raise ValueError(f"Task count invariant violated: {total} != {self.tasks_run}") return self - @computed_field + @computed_field # type: ignore[prop-decorator] @property def tasks_graded(self) -> int: """Tasks that were actually measured — the denominator for every rate below. @@ -1169,6 +1169,24 @@ def tasks_graded(self) -> int: """ return self.tasks_run - self.tasks_not_graded + @property + def _nothing_was_measured(self) -> bool: + """True when no row in this run produced a criteria verdict. + + ``tasks_graded`` is the complement of the ungraded bucket, so it still + counts ERROR rows — correct under ``run``, where an errored row was + attempted and genuinely missed. Under ``coder-eval execute`` no criterion + runs on ANY row, yet a crashed one lands in the ``error`` bucket rather + than the ``ungraded`` one, so it stayed in the denominator on its own: a + 100-task execute night with 5 crashes published ``pass_rate 0.0`` and + ``error_share 1.0`` — a measured-looking total failure for a run that was + never measured at all, and a real 0% point on the evalboard trend. + + The test is evidence again: if not one row reached a pass or a fail, the + rate has no numerator to be a fraction of. + """ + return self.tasks_not_graded > 0 and (self.tasks_succeeded + self.tasks_failed) == 0 + # Derived run metrics: computed_fields over the stored counts and # ``task_results``, so they serialize into run.json while staying impossible to # set to something the rows disagree with. Consumers should read these rather @@ -1182,7 +1200,12 @@ def pass_rate(self) -> float | None: The denominator excludes ungraded tasks (``coder-eval execute``), which were never measured — counting them as misses would report a clean execute run as 0% pass. Identical to ``tasks_run`` for every graded run. + + ``None`` also when no row produced a verdict at all, not just when the run + is empty — see ``_nothing_was_measured``. """ + if self._nothing_was_measured: + return None return self.tasks_succeeded / self.tasks_graded if self.tasks_graded else None @computed_field # type: ignore[prop-decorator] @@ -1192,8 +1215,12 @@ def error_share(self) -> float | None: Diagnostic only, never adjusts the rate: a drop at a high error share is an infrastructure night, the same drop at a normal share is the model. Shares - ``pass_rate``'s denominator so the two are directly comparable. + ``pass_rate``'s denominator so the two are directly comparable — including + being ``None`` on a run where nothing was measured, or an execute night + with one crashed row reports 100% error. """ + if self._nothing_was_measured: + return None return self.tasks_error / self.tasks_graded if self.tasks_graded else None @computed_field # type: ignore[prop-decorator] diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py index d9fe0098..57ca4684 100644 --- a/src/coder_eval/orchestration/batch.py +++ b/src/coder_eval/orchestration/batch.py @@ -28,7 +28,7 @@ TaskDefinition, TaskResult, ) -from ..path_utils import format_task_log_id +from ..path_utils import TASK_JSON_FILENAME, format_task_log_id from ..pricing import unpriced_models from ..reports_experiment import eval_result_to_task_dict from ..streaming.callbacks import StreamCallback @@ -331,7 +331,7 @@ class ResumePartition(NamedTuple): """Never finished executing — re-run from scratch.""" to_grade: list[ResolvedTask] - """Executed but ungraded (NOT_GRADED). Needs criteria, NOT another agent run.""" + """Executed but never scored. Needs criteria, NOT another agent run.""" prior_results: list[TaskResult] """Genuinely finished — reloaded so run.json covers the whole suite.""" @@ -340,6 +340,20 @@ class ResumePartition(NamedTuple): """The ResolvedTask for each entry of prior_results, same order.""" +def _owes_a_grade(result: EvaluationResult) -> bool: + """Whether a finalized row was executed but never scored. + + Evidence, not label. ``NOT_GRADED`` is the ordinary shape, but an ``execute`` + row that also tripped a run limit (TIMEOUT, a budget stop) carries an + execution-fact status whose category is ``error``/``failed`` — and no verdict + at all. Both are equally owed a grade; only the first announces it. + + A row that carries a criteria vector or a score has been graded, whatever its + status, so a genuine FAILURE/ERROR from ``run`` is untouched. + """ + return result.weighted_score is None and not result.success_criteria_results + + def partition_for_resume(resolved_tasks: list[ResolvedTask], *, grade: bool = True) -> ResumePartition: """Split resolved tasks over what ``--resume`` still owes each one. @@ -358,9 +372,20 @@ def partition_for_resume(resolved_tasks: list[ResolvedTask], *, grade: bool = Tr caller runs the criteria against the trajectory and workspace already on disk rather than paying for the agent a second time. - Note the asymmetry is only for NOT_GRADED. FAILURE and ERROR stay complete - under both commands — resume has never retried failures (delete a task's - task.json to force that), and this does not change it. + The test is the row's **evidence**, not its status: a row is owed a grade + when it was executed but never scored. Keying on ``category == "ungraded"`` + alone missed every ``execute`` row that also carries an execution fact — a + TIMEOUT or budget stop aborts the run before grading, so under ``execute`` it + lands with ``weighted_score is None`` and an empty criteria vector, yet its + category is ``error``/``failed`` and resume filed it as complete. It then + stayed permanently unscored in run.json, the rollup and the evalboard, while + ``evaluate `` graded the identical bytes happily — two entry points + into one feature disagreeing about the same record. + + Note the asymmetry is only for a row that was never scored. FAILURE and ERROR + rows that DO carry a verdict stay complete under both commands — resume has + never retried failures (delete a task's task.json to force that), and this + does not change it. Args: resolved_tasks: Fully-resolved tasks for the whole run. @@ -377,7 +402,7 @@ def partition_for_resume(resolved_tasks: list[ResolvedTask], *, grade: bool = Tr tr = _load_completed_result(rt) if tr is None: to_run.append(rt) - elif grade and tr.result.final_status.category == "ungraded": + elif grade and _owes_a_grade(tr.result): to_grade.append(rt) else: prior_results.append(tr) @@ -408,7 +433,7 @@ def clear_rerun_artifacts(to_run: list[ResolvedTask]) -> int: def _load_completed_result(rt: ResolvedTask) -> TaskResult | None: """Reconstruct a TaskResult from a finalized task.json, or None if absent/incomplete.""" - report_path = rt.run_dir / "task.json" + report_path = rt.run_dir / TASK_JSON_FILENAME try: text = report_path.read_text(encoding="utf-8") except OSError: @@ -455,7 +480,7 @@ def recover_task_results(run_dir: Path) -> list[TaskResult]: """ nested_roots = [p.parent for p in run_dir.rglob("run.json") if p.parent != run_dir] recovered: list[TaskResult] = [] - for task_json in run_dir.rglob("task.json"): + for task_json in run_dir.rglob(TASK_JSON_FILENAME): if any(root in task_json.parents for root in nested_roots): continue # belongs to a nested sub-run (its own run.json), not this one try: diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index 6c19ea14..8c48ef99 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -13,7 +13,7 @@ import re from collections.abc import Sequence from pathlib import Path -from typing import Any +from typing import Any, Literal import yaml @@ -818,8 +818,8 @@ def resolve_all_tasks( def _pick_worst_status(statuses: list[FinalStatus]) -> FinalStatus: """Pick the worst final_status across replicates (error > failed > succeeded). - Unknown categories fall back to priority -1 so they sort as worst-of-all - (fail-closed: a new unrecognised status becomes the most urgent). + Every category is enumerated below and indexed directly, so a new one is a + type error here rather than a silent placement. "ungraded" sorts LEAST urgent (above "succeeded") — it carries no verdict, so any replicate that does have one must win. It therefore survives only when @@ -832,8 +832,22 @@ def _pick_worst_status(statuses: list[FinalStatus]) -> FinalStatus: while its sibling is not, and the ordering above is what keeps that from absorbing an unmeasured replicate into a pass. """ - priority = {"error": 0, "failed": 1, "succeeded": 2, "ungraded": 3} - return min(statuses, key=lambda s: priority.get(s.category, -1)) + # Annotated with the SAME Literal `FinalStatus.category` returns, and indexed + # directly rather than via `.get(..., -1)`. Adding the fourth `ungraded` + # bucket here was a manual step no checker could verify — an untyped + # `dict[str, int]` proves neither that every category is present nor that no + # stray key is — while the `-1` default it leaned on was already unreachable + # (the `assert set(_STATUS_CATEGORIES) == set(FinalStatus)` in models/enums.py + # makes `category` total). Worse, that default was documented as + # "fail-closed", but -1 sorts BELOW error, so a fifth category would have + # silently outranked ERROR as the worst status. + priority: dict[Literal["succeeded", "failed", "error", "ungraded"], int] = { + "error": 0, + "failed": 1, + "succeeded": 2, + "ungraded": 3, + } + return min(statuses, key=lambda s: priority[s.category]) def _measured_scores(rows: Sequence[VariantResult] | Sequence[TaskResult]) -> list[float]: diff --git a/src/coder_eval/orchestration/regrade.py b/src/coder_eval/orchestration/regrade.py index 54f598d1..1b9b09cb 100644 --- a/src/coder_eval/orchestration/regrade.py +++ b/src/coder_eval/orchestration/regrade.py @@ -58,7 +58,7 @@ def task_from_prior( run_dir: Path, *, allow_recorded_commands: bool = False, - include_hooks: bool = True, + include_setup_phase: bool = True, ) -> tuple[TaskDefinition, str]: """Rebuild the executed task from the run's own recorded config. @@ -85,20 +85,32 @@ def task_from_prior( task = TaskDefinition.model_validate(record.resolved) except ValueError as e: return _fall_back_to_source( - record, run_dir, e, allow_recorded_commands=allow_recorded_commands, include_hooks=include_hooks + record, run_dir, e, allow_recorded_commands=allow_recorded_commands, include_setup_phase=include_setup_phase ) - check_embedded_commands(task, run_dir, allow_recorded_commands=allow_recorded_commands, include_hooks=include_hooks) + check_embedded_commands( + task, run_dir, allow_recorded_commands=allow_recorded_commands, include_setup_phase=include_setup_phase + ) return task, record.source_yaml -def embedded_commands(task: TaskDefinition, *, include_hooks: bool = True) -> list[str]: +def embedded_commands(task: TaskDefinition, *, include_setup_phase: bool = True) -> list[str]: """Every shell command a rebuilt task definition would run on this host. - ``include_hooks`` covers ``pre_run`` / ``post_run``. They are SKIPPED on the - in-place grading path (``Sandbox.was_adopted`` — re-running them would - overwrite the agent's deliverables before the criteria read them), so on that - path they are not a capability the run dir has; on the ``--copy`` path they - are. + ``include_setup_phase`` covers the two capability families that exist only on + the ``--copy`` path: ``pre_run`` / ``post_run``, and the sandbox's own + provisioning. Both are SKIPPED when grading in place (``Sandbox.adopt`` runs + no installer, and re-running the hooks would overwrite the agent's + deliverables before the criteria read them), so on that path they are not a + capability the run dir has. + + Sandbox provisioning is the half this gate originally missed, and it was the + worst one. ``grading_sandbox_config`` carries the recorded ``sandbox`` block + through untouched, and the ``--copy`` branch then calls ``Sandbox.setup``, + which reaches ``uv pip install ``, ``npm install `` and ``git clone ``. A package name is arbitrary + code at install time. Because the scan walked only ``success_criteria``, a + shared run directory whose criteria were all ``file_exists`` sailed through + the gate and still ran installers of the attacker's choosing. ``isinstance`` narrowing, never ``getattr(c, "command", None)``: an untyped string probe over a discriminated union is invisible to pyright, so renaming @@ -107,7 +119,13 @@ def embedded_commands(task: TaskDefinition, *, include_hooks: bool = True) -> li cannot reach ``agent_judge``, whose ``bash`` tooling is the widest blast radius of the three. """ - from coder_eval.models import AgentJudgeCriterion, RunCommandCriterion, UiPathEvalCriterion + from coder_eval.models import ( + AgentJudgeCriterion, + LLMJudgeCriterion, + RepoSource, + RunCommandCriterion, + UiPathEvalCriterion, + ) commands: list[str] = [] for c in task.success_criteria: @@ -118,18 +136,32 @@ def embedded_commands(task: TaskDefinition, *, include_hooks: bool = True) -> li # with tool access (Bash included) under the grader's credentials, # which is a strictly wider capability than one shell line. commands.append(f"") + elif isinstance(c, LLMJudgeCriterion): + # No shell, but it spends the grader's model budget and ships the + # graded artifacts (and optionally the trajectory) to a provider of + # the recorded config's choosing. That is a capability the operator + # should approve, even though nothing executes locally. + commands.append(f"") elif isinstance(c, UiPathEvalCriterion): # Builds and shells `uv run uipath eval …`. Every argument is # shlex-quoted, so this is disclosure rather than injection — but it # is still a subprocess the recorded config chose to start. commands.append(f"uv run uipath eval {c.agent_name} {c.eval_set}") - if include_hooks: + if include_setup_phase: commands += [c.command for c in task.pre_run] + [c.command for c in task.post_run] + sandbox = task.sandbox + if sandbox.python is not None and sandbox.python.env_packages: + commands.append(f"uv pip install {' '.join(sandbox.python.env_packages)}") + if sandbox.node is not None and sandbox.node.env_packages: + commands.append(f"npm install {' '.join(sandbox.node.env_packages)}") + for source in sandbox.template_sources or []: + if isinstance(source, RepoSource): + commands.append(f"git clone -- {source.url}") return commands def check_embedded_commands( - task: TaskDefinition, run_dir: Path, *, allow_recorded_commands: bool, include_hooks: bool = True + task: TaskDefinition, run_dir: Path, *, allow_recorded_commands: bool, include_setup_phase: bool = True ) -> None: """Refuse — or at minimum name — the shell a rebuilt config will run here. @@ -148,7 +180,7 @@ def check_embedded_commands( Passing the task file explicitly (``evaluate ``) also bypasses this: that config came from the operator, not from the artifact. """ - commands = embedded_commands(task, include_hooks=include_hooks) + commands = embedded_commands(task, include_setup_phase=include_setup_phase) if not commands: return rendered = "; ".join(commands) @@ -174,7 +206,7 @@ def _fall_back_to_source( e: ValueError, *, allow_recorded_commands: bool, - include_hooks: bool = True, + include_setup_phase: bool = True, ) -> tuple[TaskDefinition, str]: """The loud source-YAML fallback for a resolved config that no longer validates.""" from .task_loader import load_task @@ -192,7 +224,9 @@ def _fall_back_to_source( record.source_file, ) task, source_yaml = load_task(Path(record.source_file)) - check_embedded_commands(task, run_dir, allow_recorded_commands=allow_recorded_commands, include_hooks=include_hooks) + check_embedded_commands( + task, run_dir, allow_recorded_commands=allow_recorded_commands, include_setup_phase=include_setup_phase + ) return task, source_yaml diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 33420fe3..35adc934 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -7,7 +7,7 @@ import tempfile import time import uuid -from collections.abc import Callable +from collections.abc import Callable, Sequence from contextlib import suppress from dataclasses import dataclass from datetime import datetime @@ -68,6 +68,7 @@ from .orchestration.evaluation import resolve_reference_dir, stage_reference_dir from .orchestration.run_limits import validate_run_limits from .path_utils import ( + TASK_JSON_FILENAME, digest_tree, format_task_log_id, rmtree_restrictive, @@ -434,7 +435,7 @@ def __init__( self.prior_result = prior_result # Derived paths - self.report_path = self.run_dir / "task.json" + self.report_path = self.run_dir / TASK_JSON_FILENAME self.html_report_path = self.run_dir / "task.html" # Clean user<->agent transcript for simulation runs. Written alongside # task.log so a human can follow the conversation without the @@ -536,11 +537,25 @@ def _terminal_status(self, success: bool) -> FinalStatus: phase this pass neither repeated nor observed. Without the first arm a crashed run re-graded against its half-finished workspace reports SUCCESS — with the original ``error_message`` still attached. - * The NOT_GRADED arm sits between the execution facts and FAILURE: under - ``grade=False`` no criterion ran, so ``success`` is always False and - FAILURE would be a verdict never actually reached — but - MAX_TURNS_EXHAUSTED is a fact about the RUN, like the statuses the - ``except`` branches assign, and still applies. + * The NOT_GRADED arm sits ABOVE ``max_turns_exhausted``, and that order + is what makes ``execute`` + ``evaluate`` equal a single ``run``. + MAX_TURNS_EXHAUSTED reads like an execution fact but is not one: on the + graded path it is subordinate to the verdict — ``run`` returns SUCCESS + for a max-turns trajectory whose criteria pass, and only falls through + to MAX_TURNS_EXHAUSTED when they do not. So it is not knowable under + ``grade=False``. Consuming it here first made it terminal *and* + permanent (``is_execution_fact`` pins it in the first arm), so the same + agent output scored SUCCESS/1.0 under ``run`` and MAX_TURNS_EXHAUSTED + under ``execute`` → ``evaluate`` — and, being category ``failed``, + `run --resume` called the row complete and left it forever unscored. + Nothing is lost by deferring: the fact lives on + ``result.max_turns_exhausted``, which ``_seed_from_prior_result`` + carries, so the detached grade walks this identical chain and reaches + the arm below. + + The statuses that ARE execution facts (TIMEOUT, ERROR, the budget + stops) differ in kind: they abort the run before a verdict is + reachable, so preserving them overturns nothing. With ``grade=True`` and no prior result the chain is the original one. """ @@ -554,10 +569,10 @@ def _terminal_status(self, success: bool) -> FinalStatus: return inherited if success: return FinalStatus.SUCCESS - if self.result.max_turns_exhausted: - return FinalStatus.MAX_TURNS_EXHAUSTED if not self.grade: return FinalStatus.NOT_GRADED + if self.result.max_turns_exhausted: + return FinalStatus.MAX_TURNS_EXHAUSTED return FinalStatus.FAILURE async def run(self) -> EvaluationResult: @@ -2184,13 +2199,22 @@ def _select_gate(self) -> bool: arming a criterion (e.g. adding a ``decide_within`` fail-fast timeout) must never change the verdict of a run it didn't cut. - BOTH grading paths must call this. A detached grade (``evaluate - `` / ``run --resume``) reaches the verdict through the - evaluate-only branch, where ``early_stop`` arrives via + BOTH SINGLE-SHOT grading paths must call this — the live one and the + evaluate-only one, which are its two call sites. A detached grade + (``evaluate `` / ``run --resume``) reaches the verdict through + the evaluate-only branch, where ``early_stop`` arrives via ``_seed_from_prior_result`` rather than from a live watcher; selecting the gate there in a second, hand-written place is exactly how the seeded field came to be carried but never read — re-grading an early-stopped run under the full-run strict-AND gate flips its verdict. + + The simulation dialog path does NOT route through here, and saying "both + grading paths" without that qualifier read as though it did. It is + benign only because ``result.early_stop`` is never assigned on the + dialog path, so an armed simulation task silently gates strict-AND on a + possibly-truncated trajectory. Wiring the dialog path through this seam + means also setting ``early_stop`` there; until then the limit is stated + rather than implied. """ assert self.result is not None if self.result.early_stop is not None: @@ -2318,11 +2342,15 @@ async def _evaluation_loop(self) -> bool: logger.debug(f"Agent response received ({len(turn_record.agent_output)} chars)") # Facts about the RUN, recorded before the grading switch. `execute` - # withholds the verdict, never the facts: a max-turns or over-budget run - # must finalize the same way under `execute` as under `run`, or - # `execute` exits 0 where `run` exits 1 for identical agent output — and - # `_seed_from_prior_result` cannot restore a fact the execute phase never - # captured, so a later `evaluate` inherits the wrong terminal status too. + # withholds the verdict, never the facts: `_seed_from_prior_result` + # cannot restore a fact the execute phase never captured, so a later + # `evaluate` would inherit the wrong terminal status. + # + # Recording the fact is NOT the same as finalizing on it. `max_turns` + # exhaustion decides the status only when the criteria fail (see + # `_terminal_status`), so under `grade=False` this flag is carried into + # task.json and consumed by the detached grade, not turned into a + # terminal status here. if turn_record.max_turns_exhausted: self.result.max_turns_exhausted = True logger.warning( @@ -2424,12 +2452,24 @@ async def _run_dialog_criteria_check( # Grading site 3 of 4. Unreachable today — `execute` rejects simulation # tasks at the CLI, because the dialog's turn-continuation logic reads # criteria results to decide whether to keep talking, so an ungraded - # dialog would silently change its own stopping behavior. Kept as a - # correct, defensive no-op so the gate holds if that restriction lifts. + # dialog would silently change its own stopping behavior. + # + # RAISES rather than returning an empty list, matching the evaluate-only + # path's refusal. The empty-list version described itself as a + # "defensive no-op so the gate holds", and it was neither: both callers + # go straight on to `all_criteria_passed`, whose first act is a + # length pre-check that raises on a mismatch — and an empty criteria + # list is forbidden by `TaskDefinition.validate_success_criteria`, so + # the mismatch was guaranteed. If the simulation restriction ever lifts, + # that "no-op" turns every ungraded dialog into FinalStatus.ERROR. A + # loud refusal here is honest about the fact that this path has no + # ungraded semantics yet. if not self.grade: - self.result.success_criteria_results = [] - self.result.weighted_score = None - return [] + raise ValueError( + "Grading is disabled but the simulation dialog path requires criteria results to " + + "decide turn continuation. `execute` refuses simulation tasks at the CLI; reaching " + + "here means that refusal was bypassed." + ) await self._verify_reference_integrity() criteria_results = await self.success_checker.check_all_async( self.task.success_criteria, @@ -3031,7 +3071,7 @@ async def _run_pre_run_commands(self) -> None: Skipped entirely on an adopted sandbox — see ``_skip_hooks_for_adopted``. """ - if self.result is None or self._skip_hooks_for_adopted("pre_run"): + if self.result is None or self._skip_hooks_for_adopted(self.task.pre_run, "pre_run"): return await self._run_command_list(self.task.pre_run, self.result.pre_run_results, "pre_run") @@ -3044,11 +3084,11 @@ async def _run_post_run_commands(self) -> None: Skipped entirely on an adopted sandbox — see ``_skip_hooks_for_adopted``. """ - if self.result is None or self._skip_hooks_for_adopted("post_run"): + if self.result is None or self._skip_hooks_for_adopted(self.task.post_run, "post_run"): return await self._run_command_list(self.task.post_run, self.result.post_run_results, "post_run") - def _skip_hooks_for_adopted(self, phase: str) -> bool: + def _skip_hooks_for_adopted(self, commands: Sequence[PreRunCommand | PostRunCommand], phase: str) -> bool: """True when ``phase``'s commands must not run against an adopted sandbox. ``adopt()`` guarantees it materializes nothing into the workspace, but @@ -3061,10 +3101,15 @@ def _skip_hooks_for_adopted(self, phase: str) -> bool: The hooks belong to the EXECUTE phase; the prior run already ran them, and their recorded results are carried over by ``_seed_from_prior_result``. + + ``commands`` is passed in rather than looked up from ``phase``. The + lookup was a stringly-typed branch whose only consumer was a log line, so + a typo (``"prerun"``) would silently report the post_run count with + nothing — not pyright, not ruff — able to see it, in the same module + CE050 was written to protect from exactly that. """ if self.sandbox is None or not self.sandbox.was_adopted: return False - commands = self.task.pre_run if phase == "pre_run" else self.task.post_run if commands: logger.info( "Skipping %d %s command(s): the sandbox was adopted for grading, and re-running them " diff --git a/src/coder_eval/path_utils.py b/src/coder_eval/path_utils.py index ba99620d..7f68318c 100644 --- a/src/coder_eval/path_utils.py +++ b/src/coder_eval/path_utils.py @@ -5,6 +5,7 @@ import logging import os import platform +import secrets import shutil from collections.abc import Callable from datetime import datetime @@ -47,23 +48,40 @@ def write_text_atomic(path: Path, text: str) -> None: orchestrator and the detached grade's write-back cannot have different crash semantics for the same file. - The temp file is opened ``O_CREAT | O_EXCL | O_NOFOLLOW``. Without it a - pre-planted ``task.json.tmp`` *symlink* in a shared run directory makes this - an arbitrary-file-overwrite primitive — and one that bypasses the destination - symlink refusal in ``evaluate``'s write-back, since the guard checks the - destination while the truncation happens through the temp name. A partial - temp file is unlinked before the error propagates, so a failed write never - leaves ``.tmp`` litter beside the record. + The temp file is opened ``O_CREAT | O_EXCL | O_NOFOLLOW`` under a name that + is UNIQUE per call. Without ``O_NOFOLLOW`` a pre-planted ``task.json.tmp`` + *symlink* in a shared run directory makes this an arbitrary-file-overwrite + primitive — and one that bypasses the destination symlink refusal in + ``evaluate``'s write-back, since the guard checks the destination while the + truncation happens through the temp name. + + The name must be unique, not fixed, and that is a correctness requirement + rather than tidiness. ``os.replace`` is the only step that can be interrupted + without trace, and this function exists precisely because the process may be + SIGKILLed (the docker host-heartbeat watchdog does exactly that) — so a + crash between ``open`` and ``replace`` WILL sometimes leave the temp file + behind. Under a fixed name, ``O_EXCL`` then turned that leftover into a + permanent refusal to write the record at all: the row reported ERROR, and + ``--resume`` saw no ``task.json``, re-ran the task into the same run dir, and + hit the same stale file — an unbounded loop that re-pays for the agent every + time. A unique name keeps ``O_EXCL``'s guarantee while making a leftover + inert. It can litter a dead ``.tmp`` beside the record after a hard kill; + that is strictly better than wedging finalization, and the litter is + recognisable by its embedded pid. + + Mode is ``0o666`` so the umask applies, giving the same 0644 a plain + ``write_text`` produced. Creating it 0600 broke the docker driver on Linux: + the in-container orchestrator writes ``task.json`` as root straight into the + bind-mounted host run dir, and the host then reads it back as the invoking + uid — an unguarded read that raises ``PermissionError`` for every task. A + result record is not a secret, and the symlink hazard is closed by + ``O_NOFOLLOW`` and the unpredictable name rather than by the mode. """ - tmp = path.with_suffix(path.suffix + ".tmp") + # pid + random: unique across concurrent writers AND across a crashed + # predecessor, so O_EXCL can never collide with our own leftovers. + tmp = path.with_name(f"{path.name}.{os.getpid()}.{secrets.token_hex(4)}.tmp") flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0) - try: - fd = os.open(tmp, flags, 0o600) - except FileExistsError as e: - raise OSError( - f"Refusing to write {path}: {tmp} already exists. Remove it if it is stale — a " - + "pre-planted temp file (especially a symlink) would redirect this write." - ) from e + fd = os.open(tmp, flags, 0o666) try: with os.fdopen(fd, "w", encoding="utf-8") as fh: fh.write(text) diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py index 4762ea65..d04488d0 100644 --- a/src/coder_eval/reports.py +++ b/src/coder_eval/reports.py @@ -20,7 +20,7 @@ row_cost_incomplete, sum_costs, ) -from .path_utils import build_task_run_dir +from .path_utils import TASK_JSON_FILENAME, build_task_run_dir if TYPE_CHECKING: @@ -244,7 +244,15 @@ def _pass_rate_lines(summary: RunSummary) -> list[str]: """ # Only an ungraded run gets the explanatory line. An ordinary EMPTY run keeps # its original "n/a (0/0)" rendering — the two are different facts. - if summary.tasks_not_graded and not summary.tasks_graded: + # + # `pass_rate is None` rather than `not tasks_graded`: an execute night with a + # crashed row has tasks_graded > 0 (an ERROR row is category `error`, not + # `ungraded`, so it stays in the denominator) while still having measured + # nothing — and this line then rendered `0.0% (0/5)` plus `Error Share: + # 100.0%`, exactly the total-failure reading the guard exists to prevent. + # Deferring to the model keeps one rule for run.md, run.json and the + # evalboard instead of three. + if summary.tasks_not_graded and summary.pass_rate is None: return [f"- **Pass Rate**: n/a — {summary.tasks_not_graded} task(s) executed without grading"] lines = [f"- **Pass Rate**: {_fmt_rate(summary.pass_rate)} ({summary.tasks_succeeded}/{summary.tasks_graded})"] if summary.tasks_error: @@ -734,7 +742,7 @@ def _aggregate_command_statistics(run_dir: Path) -> CommandStatistics | None: all_turns: list[TurnRecord] = [] # Find all task.json files recursively to handle both flat and nested (experiment) layouts - for report_path in run_dir.rglob("task.json"): + for report_path in run_dir.rglob(TASK_JSON_FILENAME): if "artifacts" in report_path.parts or ".git" in report_path.parts: continue try: @@ -965,7 +973,8 @@ def _compute_suite_rollup( if len(reasons) >= _FAILURE_REASONS_PER_ROW: break task_json_path = ( - build_task_run_dir(run_dir, variant_id, row.task_id, replicate_index=row.replicate_index) / "task.json" + build_task_run_dir(run_dir, variant_id, row.task_id, replicate_index=row.replicate_index) + / TASK_JSON_FILENAME ) try: # Persist as POSIX — this value lands in suite.json and in diff --git a/src/coder_eval/reports_junit.py b/src/coder_eval/reports_junit.py index acc609a7..069dd53d 100644 --- a/src/coder_eval/reports_junit.py +++ b/src/coder_eval/reports_junit.py @@ -29,6 +29,7 @@ from .evaluation.judge_context import truncate from .models import FinalStatus, RunSummary, SuiteRollup +from .path_utils import TASK_JSON_FILENAME logger = logging.getLogger(__name__) @@ -169,14 +170,14 @@ def _load_task_json(run_dir: Path, row: dict[str, Any], variant: str) -> dict[st replicate_index = row.get("replicate_index") task_dir = run_dir / variant / task_id if isinstance(replicate_index, int) and not isinstance(replicate_index, bool): - candidate = task_dir / f"{replicate_index:02d}" / "task.json" + candidate = task_dir / f"{replicate_index:02d}" / TASK_JSON_FILENAME else: - matches = sorted(task_dir.glob("*/task.json")) + matches = sorted(task_dir.glob(f"*/{TASK_JSON_FILENAME}")) # With no replicate index, picking one of several would misattribute # another replicate's failure detail to this row — degrade instead. if len(matches) > 1: return None - candidate = matches[0] if matches else task_dir / "task.json" + candidate = matches[0] if matches else task_dir / TASK_JSON_FILENAME try: # Belt-and-braces containment check (catches symlink escapes too). diff --git a/src/coder_eval/reports_stats.py b/src/coder_eval/reports_stats.py index 18ee2751..ba9f9de3 100644 --- a/src/coder_eval/reports_stats.py +++ b/src/coder_eval/reports_stats.py @@ -16,6 +16,8 @@ from coder_eval.models import EvaluationResult, ExperimentResult, ExperimentVariant, TaskExperimentSummary +from .path_utils import TASK_JSON_FILENAME + logger = logging.getLogger(__name__) @@ -527,7 +529,7 @@ def load_variant_eval_results( if not task_dir.is_dir(): continue for rep_subdir in sorted(task_dir.glob("[0-9][0-9]")): - task_json = rep_subdir / "task.json" + task_json = rep_subdir / TASK_JSON_FILENAME if task_json.exists(): try: results.append(EvaluationResult.model_validate_json(task_json.read_text(encoding="utf-8"))) diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index b77bdb60..06127eda 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -451,7 +451,13 @@ def _apply_repo_source(self, source: RepoSource) -> None: assert self.sandbox_dir is not None, "Sandbox directory not initialized" repo_dir = self.sandbox_dir / "repo" - cmd = ["git", "clone", source.url, str(repo_dir)] + # `--` before the URL: it is argv position 2, so without the separator a + # value beginning with `-` is parsed by git as an OPTION rather than a + # repository (`--upload-pack=…` runs a command of the caller's choosing). + # That URL is task-authored, and since `evaluate ` rebuilds the + # task from a shareable run directory it is no longer necessarily the + # operator's own string. + cmd = ["git", "clone", "--", source.url, str(repo_dir)] try: subprocess.run(cmd, check=True, capture_output=True, text=True, encoding="utf-8", timeout=60) @@ -1290,6 +1296,39 @@ def run_command(self, command: str, timeout: float | int | None = None) -> tuple # needs filesystem access beyond the sandbox root (e.g., reading installed packages, # system headers). Path traversal protection is handled at the agent permission level. + def _within_sandbox(self, candidate: Path) -> bool: + """Whether a resolved criterion path stays inside the sandbox. + + The read-side twin of :meth:`_resolve_within_sandbox`, which every OTHER + task-authored path already goes through. Criterion paths were the one + consumer that skipped it, and ``Path('/tmp/sandbox') / '/etc/passwd'`` is + ``/etc/passwd`` — pathlib discards the prefix on an absolute right + operand — so ``file_contains`` / ``file_check`` / ``file_matches_regex`` + were a pass-fail oracle over any file the grading user could read, and + ``json_check`` could surface parsed values in ``details``. + + That was defensible while a task YAML was operator-supplied. It stopped + being so when ``evaluate `` began rebuilding the criteria list + from a shareable run directory. + + Returns False rather than raising: an out-of-sandbox path is + indistinguishable to the criterion from a file that is not there, which + is the same answer the template and mock-dir paths give, and raising + here would book a config error as an agent crash (CE039). + """ + assert self.sandbox_dir is not None + root = self.sandbox_dir.resolve() + try: + resolved = candidate.resolve() + except OSError: + return False + if resolved == root or root in resolved.parents: + return True + logger.warning( + "Criterion path %r resolves outside the sandbox (%s); treating it as no match.", str(candidate), root + ) + return False + def resolve_files(self, path: str) -> list[Path]: """Resolve a criterion ``path`` to the sandbox files it addresses. @@ -1324,7 +1363,7 @@ def resolve_files(self, path: str) -> list[Path]: # Literal first: an existing path is never reinterpreted as a pattern. candidate = self.sandbox_dir / path if candidate.exists(): - return [candidate] + return [candidate] if self._within_sandbox(candidate) else [] if not _is_glob(path): return [] @@ -1334,7 +1373,7 @@ def resolve_files(self, path: str) -> list[Path]: matches: list[Path] = [] for match in self.sandbox_dir.glob(path): - if not match.is_file(): + if not match.is_file() or not self._within_sandbox(match): continue discovered = [part for part in match.relative_to(self.sandbox_dir).parts if part not in pinned] if discovered and should_ignore_path(Path(*discovered), patterns): diff --git a/tests/lint/rules/ce053_run_record_filename_literal.py b/tests/lint/rules/ce053_run_record_filename_literal.py new file mode 100644 index 00000000..41d30aa5 --- /dev/null +++ b/tests/lint/rules/ce053_run_record_filename_literal.py @@ -0,0 +1,70 @@ +"""CE053: no bare run-record filename literal outside ``path_utils``. + +``path_utils`` defines ``TASK_JSON_FILENAME`` / ``PRE_GRADE_JSON_FILENAME`` and +its comment states why: "~12 sites name them — including three that ``rglob`` for +the first — and two half-copies of the same string in different packages is how a +rename becomes a silent no-op on the sites it missed." + +The constant shipped with that rationale and the twelve pre-existing literals +were not converted, so it created exactly the second source of truth it argues +against and delivered zero rename safety: the new modules used the constant, and +``orchestrator.py``, ``batch.py``, ``docker_runner.py``, ``reports.py``, +``reports_junit.py``, ``reports_stats.py`` and ``report_command.py`` kept the +string — the three ``rglob("task.json")`` calls the comment specifically cites +among them. + +A rationale that only a human remembers is not a rule. Fires on any string +constant in ``src/coder_eval/`` (outside ``path_utils.py``) that equals one of +those filenames, or embeds it as a trailing path segment (``"*/task.json"``). +Import the constant instead; ``# noqa: CE053`` for a genuinely unrelated string. +""" + +import ast +import re + +from tests.lint.rules.base import BaseRule + + +def _run_record_filenames() -> set[str]: + """The filenames from ``path_utils``, read from the module rather than retyped.""" + from coder_eval.path_utils import PRE_GRADE_JSON_FILENAME, TASK_JSON_FILENAME + + return {TASK_JSON_FILENAME, PRE_GRADE_JSON_FILENAME} + + +class NoRunRecordFilenameLiteral(BaseRule): + id = "CE053" + + # `(^|sep)` so a repo-relative path is in scope too; see CE047. + _SRC_PATH = re.compile(r"(?:^|[/\\])src[/\\]coder_eval[/\\]") + # The module that DEFINES them, and the container-path module that mirrors + # the in-container layout as its own vocabulary. + _EXEMPT = re.compile(r"[/\\]path_utils\.py$") + _names: set[str] | None = None + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = bool(self._SRC_PATH.search(filepath)) and not self._EXEMPT.search(filepath) + if self._in_scope and NoRunRecordFilenameLiteral._names is None: + NoRunRecordFilenameLiteral._names = _run_record_filenames() + + def visit_Constant(self, node: ast.Constant) -> None: + if self._in_scope and isinstance(node.value, str): + self._check(node, node.value) + self.generic_visit(node) + + def _check(self, node: ast.Constant, value: str) -> None: + for name in NoRunRecordFilenameLiteral._names or set(): + # Exact, or a glob/path whose LAST segment is the filename. Not a + # bare `in`: that would fire on prose in a docstring or an error + # message, where naming the file is the point. + if value == name or (("/" in value) and value.rsplit("/", 1)[-1] == name): + self.violation( + node, + f"{value!r} names a run record by literal. path_utils exports a constant for it " + "precisely so a rename cannot leave half the tree behind — and the constant " + "shipped while twelve sites kept the string, which is the second source of " + "truth it was added to prevent. Import TASK_JSON_FILENAME / " + "PRE_GRADE_JSON_FILENAME, or add `# noqa: CE053` if the string is unrelated.", + ) + return diff --git a/tests/lint/runner.py b/tests/lint/runner.py index 8bfb6846..bcd51561 100644 --- a/tests/lint/runner.py +++ b/tests/lint/runner.py @@ -32,6 +32,7 @@ from tests.lint.rules.ce050_no_union_getattr_probe import NoUnionGetattrProbe from tests.lint.rules.ce051_no_driver_override import NoDriverOverride from tests.lint.rules.ce052_process_lethal_must_be_container_gated import ProcessLethalMustBeContainerGated +from tests.lint.rules.ce053_run_record_filename_literal import NoRunRecordFilenameLiteral from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore @@ -87,6 +88,7 @@ NoUnionGetattrProbe, NoDriverOverride, ProcessLethalMustBeContainerGated, + NoRunRecordFilenameLiteral, ] # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 2c65a278..900a7ff1 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -3818,3 +3818,86 @@ def test_the_real_module_is_clean(self): source = path.read_text(encoding="utf-8") assert not self._run(source, filepath=str(path)) assert "_os._exit(137)" in source, "the guarded call must still exist" + + +class TestRuffExternalCoversEveryRule: + """Every CE rule's documented `# noqa` must be accepted by ruff. + + `[tool.ruff.lint] external` is what stops ruff reporting RUF102 "Invalid + rule code" for a suppression it does not own. It was hand-maintained and had + fallen ~14 ids behind — including CE047 and CE048, whose own docstrings + advertise `# noqa: CE047` / `# noqa: CE048` as the supported escape hatch. So + the first person to use the documented exemption got a red `make check` + instead, for doing exactly what the rule told them to. + """ + + @staticmethod + def _external() -> set[str]: + import tomllib + from pathlib import Path + + data = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) + return set(data["tool"]["ruff"]["lint"]["external"]) + + def test_every_registered_rule_is_listed(self): + from tests.lint.runner import ALL_RULES + + missing = sorted({r.id for r in ALL_RULES} - self._external()) + assert not missing, f"add to [tool.ruff.lint] external in pyproject.toml: {missing}" + + def test_every_listed_id_is_well_formed(self): + """Cheap guard against a typo silently widening the allowlist.""" + import re + + bad = sorted(i for i in self._external() if not re.fullmatch(r"CE\d{3}", i)) + assert not bad, f"not a CE rule id: {bad}" + + +class TestCE053NoRunRecordFilenameLiteral: + """CE053 flags a `task.json` literal outside path_utils.""" + + @staticmethod + def _run(src: str, filepath: str = "src/coder_eval/reports.py"): + import ast + + from tests.lint.rules.ce053_run_record_filename_literal import NoRunRecordFilenameLiteral + + return NoRunRecordFilenameLiteral(filepath).check(ast.parse(src)) + + def test_flags_a_bare_literal(self): + assert self._run('path = run_dir / "task.json"') + + def test_flags_an_rglob(self): + """The three rglob sites are the ones the constant's own comment cites.""" + assert self._run('for p in run_dir.rglob("task.json"): pass') + + def test_flags_a_trailing_path_segment(self): + assert self._run('matches = sorted(d.glob("*/task.json"))') + + def test_flags_the_pre_grade_record_too(self): + assert self._run('backup = run_dir / "task.execute.json"') + + def test_allows_the_constant(self): + assert not self._run("path = run_dir / TASK_JSON_FILENAME") + + def test_allows_prose_that_merely_mentions_the_file(self): + """Naming the file in an error message is the point of the message.""" + assert not self._run('raise ValueError("no task.json in that directory")') + + def test_is_out_of_scope_in_path_utils(self): + assert not self._run('TASK_JSON_FILENAME = "task.json"', filepath="src/coder_eval/path_utils.py") + + def test_the_tree_is_clean(self): + """The migration must actually have happened — a rule whose only + evidence is synthetic proves nothing about the repo.""" + import ast + from pathlib import Path + + from tests.lint.rules.ce053_run_record_filename_literal import NoRunRecordFilenameLiteral + + offenders = [] + for path in Path("src/coder_eval").rglob("*.py"): + tree = ast.parse(path.read_text(encoding="utf-8")) + if NoRunRecordFilenameLiteral(str(path)).check(tree): + offenders.append(str(path)) + assert not offenders, offenders diff --git a/tests/test_detached_grading_boundaries.py b/tests/test_detached_grading_boundaries.py index 3c5bb5af..922aa553 100644 --- a/tests/test_detached_grading_boundaries.py +++ b/tests/test_detached_grading_boundaries.py @@ -14,6 +14,7 @@ from __future__ import annotations import json +import os from datetime import datetime from pathlib import Path from unittest.mock import MagicMock, patch @@ -196,15 +197,11 @@ def test_a_non_boolean_grade_is_a_hard_error(self, tmp_path: Path) -> None: assert result.exit_code == 2 assert "must be a boolean" in result.output - def test_the_default_is_to_grade(self) -> None: - """A host predating `execute` writes no key; the container must keep its - original behaviour rather than silently withholding verdicts.""" - import inspect - - from coder_eval.cli.run_task_internal_command import run_task_internal_command - - source = inspect.getsource(inspect.getmodule(run_task_internal_command)) # type: ignore[arg-type] - assert 'context.get("grade", True)' in source + # The in-container default is asserted BEHAVIOURALLY by + # `TestGradePlumbedIntoTheContainerOrchestrator::test_an_absent_key_still_grades`. + # It used to be a `assert 'context.get("grade", True)' in source` grep, which + # is the same static check that already failed here once: it passes happily + # while the line it describes is never executed. # -------------------------------------------------------------------------- @@ -331,28 +328,64 @@ async def test_an_unreadable_row_still_appears_as_ungraded(self, tmp_path: Path) class TestAtomicWriteIsNotAnOverwritePrimitive: """``evaluate``'s write-back guards its DESTINATION against a symlink, but the - truncation happens through the temp name — so a pre-planted - ``task.json.tmp`` symlink bypassed the guard entirely.""" + truncation happens through the temp name — so a pre-planted temp symlink + bypassed the guard entirely. The name is now unpredictable AND ``O_NOFOLLOW``, + so both halves are closed.""" - def test_a_pre_planted_temp_symlink_is_refused(self, tmp_path: Path) -> None: - from coder_eval.path_utils import write_text_atomic + def test_a_symlink_at_the_temp_name_is_never_followed(self, tmp_path: Path) -> None: + """Pinned by pinning the random half, since a real attacker cannot guess + it — the point is that O_NOFOLLOW still refuses even if they could.""" + from coder_eval import path_utils victim = tmp_path / "victim" victim.write_text("keep me", encoding="utf-8") target = tmp_path / "task.json" - (tmp_path / "task.json.tmp").symlink_to(victim) + planted = tmp_path / f"task.json.{os.getpid()}.deadbeef.tmp" + planted.symlink_to(victim) - with pytest.raises(OSError, match="already exists"): - write_text_atomic(target, "attacker content") + with ( + patch.object(path_utils.secrets, "token_hex", return_value="deadbeef"), + pytest.raises(OSError), + ): + path_utils.write_text_atomic(target, "attacker content") assert victim.read_text(encoding="utf-8") == "keep me" + def test_a_stale_temp_file_does_not_wedge_the_write(self, tmp_path: Path) -> None: + """The regression that mattered most. Under a FIXED temp name, `O_EXCL` + turned a leftover from a SIGKILL into a permanent refusal to persist the + record — so the row reported ERROR, `--resume` saw no task.json, re-ran + the task into the same run dir, and hit the same file again, re-paying + for the agent on every pass.""" + from coder_eval.path_utils import write_text_atomic + + target = tmp_path / "task.json" + (tmp_path / "task.json.tmp").write_text("leftover from a hard kill", encoding="utf-8") + + write_text_atomic(target, "hello") + + assert target.read_text(encoding="utf-8") == "hello" + + def test_the_record_is_readable_by_other_uids(self, tmp_path: Path) -> None: + """Under `driver: docker` the in-container orchestrator writes this file + as root straight into the bind-mounted host run dir, and the host reads it + back as the invoking uid through an UNGUARDED `read_text`. Creating it + 0600 made that raise PermissionError for every docker-driver task on + Linux — invisible on macOS, where Docker Desktop remaps ownership.""" + from coder_eval.path_utils import write_text_atomic + + target = tmp_path / "task.json" + write_text_atomic(target, "hello") + + mode = target.stat().st_mode & 0o777 + assert mode & 0o044, f"task.json is not group/other-readable: {mode:#o}" + def test_an_ordinary_write_still_works(self, tmp_path: Path) -> None: from coder_eval.path_utils import write_text_atomic target = tmp_path / "task.json" write_text_atomic(target, "hello") assert target.read_text(encoding="utf-8") == "hello" - assert not (tmp_path / "task.json.tmp").exists() + assert not list(tmp_path.glob("*.tmp")) def test_a_failed_write_leaves_no_temp_file(self, tmp_path: Path) -> None: from coder_eval.path_utils import write_text_atomic @@ -360,7 +393,7 @@ def test_a_failed_write_leaves_no_temp_file(self, tmp_path: Path) -> None: target = tmp_path / "task.json" with patch("os.replace", side_effect=OSError("boom")), pytest.raises(OSError): write_text_atomic(target, "hello") - assert not (tmp_path / "task.json.tmp").exists() + assert not list(tmp_path.glob("*.tmp")) # -------------------------------------------------------------------------- @@ -442,3 +475,200 @@ def test_a_reference_that_vanished_is_refused_not_skipped(tmp_path: Path) -> Non pytest.raises(RegradeError, match="is gone"), ): verify_reference_unchanged(prior, task, tmp_path / "t.yaml") + + +# -------------------------------------------------------------------------- +# The recorded-config gate must cover PROVISIONING, not just criteria +# -------------------------------------------------------------------------- + + +class TestRecordedProvisioningIsGatedToo: + """`grading_sandbox_config` carries the recorded `sandbox` block through + untouched, and the `--copy` branch then calls `Sandbox.setup` — which reaches + `uv pip install `, `npm install ` and + `git clone `. A package name is arbitrary code at install time. + + The gate walked only `success_criteria` + hooks, so a shared run directory + whose criteria were all `file_exists` passed it and still ran installers. + """ + + @staticmethod + def _task_with(**sandbox_kwargs) -> TaskDefinition: + return TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + sandbox=SandboxConfig(**sandbox_kwargs), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + + def test_python_packages_are_named(self) -> None: + from coder_eval.models import PythonEnvConfig + from coder_eval.orchestration.regrade import embedded_commands + + task = self._task_with(python=PythonEnvConfig(env_packages=["attacker-pkg"])) + assert any("attacker-pkg" in c for c in embedded_commands(task)) + + def test_node_packages_are_named(self) -> None: + from coder_eval.models import NodeEnvConfig + from coder_eval.orchestration.regrade import embedded_commands + + task = self._task_with(node=NodeEnvConfig(env_packages=["evil-npm"])) + assert any("evil-npm" in c for c in embedded_commands(task)) + + def test_a_repo_source_url_is_named(self) -> None: + from coder_eval.models import RepoSource + from coder_eval.orchestration.regrade import embedded_commands + + task = self._task_with(template_sources=[RepoSource(url="https://evil.example/repo.git")]) + assert any("evil.example" in c for c in embedded_commands(task)) + + def test_provisioning_alone_triggers_the_refusal(self, tmp_path: Path) -> None: + """The exact repro: every criterion is a file_exists, so the old scan + found nothing and the installers ran with no opt-in.""" + from coder_eval.models import PythonEnvConfig + from coder_eval.orchestration.regrade import check_embedded_commands + + task = self._task_with(python=PythonEnvConfig(env_packages=["attacker-pkg"])) + with pytest.raises(RegradeError, match="--allow-recorded-commands"): + check_embedded_commands(task, tmp_path, allow_recorded_commands=False) + + def test_the_in_place_path_is_exempt(self, tmp_path: Path) -> None: + """`adopt()` installs nothing, so in place these are not capabilities the + run dir has — and refusing there would break the headline flow.""" + from coder_eval.models import PythonEnvConfig + from coder_eval.orchestration.regrade import check_embedded_commands + + task = self._task_with(python=PythonEnvConfig(env_packages=["attacker-pkg"])) + check_embedded_commands(task, tmp_path, allow_recorded_commands=False, include_setup_phase=False) + + def test_an_llm_judge_is_named(self) -> None: + """No shell, but it spends the grader's model budget and ships the graded + artifacts to a provider the recorded config chose.""" + from coder_eval.models import LLMJudgeCriterion + from coder_eval.orchestration.regrade import embedded_commands + + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[LLMJudgeCriterion(description="x", prompt="grade it")], + ) + assert any("llm_judge" in c for c in embedded_commands(task)) + + +class TestGradePlumbedIntoTheContainerOrchestrator: + """The `grade` value reaching the in-container Orchestrator was asserted only + by grepping the module's own source text, which passes while the line it + describes is never executed — deleting `grade=grade` left every test green.""" + + @staticmethod + def _invoke(tmp_path: Path, context: dict) -> object: + captured: dict[str, object] = {} + + class _FakeOrchestrator: + def __init__(self, **kwargs): + captured.update(kwargs) + + async def run(self): + return _result() + + input_dir = tmp_path / "input" + input_dir.mkdir() + task_yaml = ( + "task_id: t\ndescription: d\nagent:\n type: none\n" + "success_criteria:\n - type: file_exists\n path: p.txt\n description: x\n" + ) + (input_dir / "task.yaml").write_text(task_yaml, encoding="utf-8") + (input_dir / "context.json").write_text( + json.dumps({"variant_id": "default", "source_yaml": task_yaml, **context}), encoding="utf-8" + ) + with patch("coder_eval.orchestrator.Orchestrator", _FakeOrchestrator): + runner.invoke( + app, + ["_run-task-internal", "--input", str(input_dir), "--output", str(tmp_path / "out")], + ) + return captured.get("grade") + + def test_execute_forwards_grade_false(self, tmp_path: Path) -> None: + assert self._invoke(tmp_path, {"grade": False}) is False + + def test_an_absent_key_still_grades(self, tmp_path: Path) -> None: + """A host predating `execute` writes no key; the container must keep its + original behaviour rather than silently withholding verdicts.""" + assert self._invoke(tmp_path, {}) is True + + +class TestContainerContextIsValidated: + """`json.loads` returns Any, so an annotation at this boundary reads like a + guarantee and enforces nothing.""" + + @staticmethod + def _run(tmp_path: Path, context: dict): + input_dir = tmp_path / "input" + input_dir.mkdir() + (input_dir / "task.yaml").write_text("task_id: t\n", encoding="utf-8") + (input_dir / "context.json").write_text(json.dumps(context), encoding="utf-8") + return runner.invoke(app, ["_run-task-internal", "--input", str(input_dir), "--output", str(tmp_path / "out")]) + + def test_a_non_string_variant_id_is_refused(self, tmp_path: Path) -> None: + result = self._run(tmp_path, {"variant_id": 7, "source_yaml": "task_id: t\n"}) + assert result.exit_code == 2 + assert "variant_id" in result.output + + def test_a_string_replicate_index_is_refused(self, tmp_path: Path) -> None: + """`"00"` would reach build_task_run_dir typed as int.""" + result = self._run(tmp_path, {"variant_id": "default", "replicate_index": "00", "source_yaml": "task_id: t\n"}) + assert result.exit_code == 2 + assert "replicate_index" in result.output + + +class TestCriterionPathsCannotEscapeTheSandbox: + """`Path('/tmp/sandbox') / '/etc/passwd'` is `/etc/passwd` — pathlib discards + the prefix on an absolute right operand — and criterion paths were the one + task-authored path in sandbox.py that skipped `_resolve_within_sandbox`. + + Defensible while a task YAML was operator-supplied; not once + `evaluate ` began rebuilding the criteria list from a shareable run + directory, which turns `file_contains` / `file_check` / `file_matches_regex` + into a pass-fail oracle over any file the grading user can read. + """ + + @staticmethod + def _sandbox(tmp_path: Path): + from coder_eval.sandbox import Sandbox + + sandbox = Sandbox(SandboxConfig(driver="tempdir"), task_id="t") + work = tmp_path / "work" + work.mkdir() + sandbox.sandbox_dir = work + return sandbox, work + + def test_an_absolute_path_resolves_to_nothing(self, tmp_path: Path) -> None: + sandbox, _ = self._sandbox(tmp_path) + secret = tmp_path / "secret.txt" + secret.write_text("token", encoding="utf-8") + + assert sandbox.resolve_files(str(secret)) == [] + + def test_a_dotdot_traversal_resolves_to_nothing(self, tmp_path: Path) -> None: + sandbox, _ = self._sandbox(tmp_path) + (tmp_path / "secret.txt").write_text("token", encoding="utf-8") + + assert sandbox.resolve_files("../secret.txt") == [] + + def test_a_glob_cannot_escape_either(self, tmp_path: Path) -> None: + sandbox, _ = self._sandbox(tmp_path) + (tmp_path / "secret.txt").write_text("token", encoding="utf-8") + + assert sandbox.resolve_files("../*.txt") == [] + + def test_an_ordinary_in_sandbox_path_still_resolves(self, tmp_path: Path) -> None: + """The control — the guard must not break normal grading.""" + sandbox, work = self._sandbox(tmp_path) + (work / "proof.txt").write_text("ok", encoding="utf-8") + + assert sandbox.resolve_files("proof.txt") == [work / "proof.txt"] + assert sandbox.resolve_files("*.txt") == [work / "proof.txt"] diff --git a/tests/test_execute_command.py b/tests/test_execute_command.py index ae124104..9a015431 100644 --- a/tests/test_execute_command.py +++ b/tests/test_execute_command.py @@ -282,18 +282,12 @@ async def test_docker_forwards_grade_to_the_container(tmp_path: Path, grade: boo assert (await _staged_context(tmp_path, grade=grade))["grade"] is grade -def test_container_defaults_to_grading_when_the_host_sends_no_key() -> None: - """A host predating `execute` writes no `grade` key; the container must keep - its original (grading) behavior rather than silently withholding verdicts.""" - # The parse is inline in a Typer command that cannot run outside a container, - # so this reads its source. Resolved off the function object because - # `coder_eval.cli` rebinds the submodule's name to the function it exports. - import inspect - - from coder_eval.cli.run_task_internal_command import run_task_internal_command - - source = inspect.getsource(inspect.getmodule(run_task_internal_command)) # type: ignore[arg-type] - assert 'context.get("grade", True)' in source, "the in-container default must be True (grade)" +# The in-container grading default is asserted behaviourally in +# tests/test_detached_grading_boundaries.py +# (`TestGradePlumbedIntoTheContainerOrchestrator`), which drives the real command +# with `Orchestrator` patched and reads the captured `grade` kwarg. The source-text +# grep that used to live here was duplicated verbatim in that file and proved +# nothing: deleting `grade=grade` at the call site left both greps green. def test_execute_help_explains_the_refused_flags() -> None: diff --git a/tests/test_execute_evaluate_loop.py b/tests/test_execute_evaluate_loop.py index bdd62c1f..bed20a62 100644 --- a/tests/test_execute_evaluate_loop.py +++ b/tests/test_execute_evaluate_loop.py @@ -42,9 +42,18 @@ def _row(task_dir: Path, name: str = "task.json") -> dict[str, Any]: return json.loads((task_dir / name).read_text(encoding="utf-8")) -def _invoke(args: list[str]) -> Any: +def _invoke(args: list[str], *, expect_exit: int = 0) -> Any: + """Run a CLI command and pin its exit code. + + ``expect_exit`` is explicit rather than "0 unless it raised" because the exit + code IS the contract for a CI wrapper: a helper that always demanded 0 once + pinned a preserved-TIMEOUT row exiting 0 under "All criteria passed" as the + expected behaviour. + """ result = runner.invoke(app, args) - assert result.exit_code == 0, f"{args} failed:\n{result.output}" + assert result.exit_code == expect_exit, ( + f"{args} exited {result.exit_code}, expected {expect_exit}:\n{result.output}" + ) return result @@ -328,7 +337,12 @@ def test_evaluate_grades_the_directory_named_by_workspace(tmp_path: Path) -> Non def test_evaluate_refuses_to_re_grade_a_run_that_errored(tmp_path: Path) -> None: """Grading may only move NOT_GRADED to a verdict. An ERROR / TIMEOUT run is an execution fact this pass neither repeated nor observed — laundering it - into SUCCESS would report a crashed run as a pass.""" + into SUCCESS would report a crashed run as a pass. + + The exit code has to agree. It exited 0 under "All criteria passed! ✓" for a + row `run.json` counts as failed, because the gate read the criteria tally + rather than the outcome — so a CI wrapper shelling `coder-eval evaluate` + went green on a timed-out run.""" run_dir = tmp_path / "r" _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) task_dir = _task_dir(run_dir) @@ -336,9 +350,32 @@ def test_evaluate_refuses_to_re_grade_a_run_that_errored(tmp_path: Path) -> None row["final_status"] = FinalStatus.TIMEOUT.value (task_dir / "task.json").write_text(json.dumps(row), encoding="utf-8") - _invoke(["evaluate", str(task_dir)]) + result = _invoke(["evaluate", str(task_dir)], expect_exit=1) assert _row(task_dir)["final_status"] == FinalStatus.TIMEOUT.value + assert "All criteria passed" not in result.output + assert FinalStatus.TIMEOUT.value in result.output + + +def test_an_inherited_error_still_renders_its_criteria_and_keeps_the_status(tmp_path: Path) -> None: + """The other arm of the same confusion. A PRESERVED ERROR is not a grading + crash: the ERROR branch fired anyway, printed the ORIGINAL run's crash + message as though grading had failed, claimed the row was "left ungraded" + (it was not — the restored record still reads ERROR), and discarded a verdict + it had just computed.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + row = _row(task_dir) + row["final_status"] = FinalStatus.ERROR.value + row["error_message"] = "agent crashed during the original run" + (task_dir / "task.json").write_text(json.dumps(row), encoding="utf-8") + + result = _invoke(["evaluate", str(task_dir)], expect_exit=1) + + assert "Criteria Results" in result.output, "the computed verdict was thrown away" + assert "left ungraded" not in result.output, "grading did not crash; saying so is false" + assert _row(task_dir)["final_status"] == FinalStatus.ERROR.value def test_grading_the_same_run_twice_reaches_the_same_verdict(tmp_path: Path) -> None: @@ -401,7 +438,22 @@ def _run(command: str, run_dir: Path) -> Any: assert executed["max_turns_exhausted"] is True, ( "`execute` dropped a fact about the run. Only the verdict is withheld." ) - assert executed["final_status"] == FinalStatus.MAX_TURNS_EXHAUSTED.value + # The FACT is recorded; the STATUS is not decided. `run` returns SUCCESS for + # a max-turns trajectory whose criteria pass and only falls through to + # MAX_TURNS_EXHAUSTED when they fail — so the status is not knowable without + # grading, and claiming it here made it both terminal and permanent + # (MAX_TURNS_EXHAUSTED is an execution fact, which the detached grade may + # never overturn). + assert executed["final_status"] == FinalStatus.NOT_GRADED.value + + # The parity that matters: grading the executed run must land exactly where + # `run` did. Asserting only the executed half is what let the divergence ship. + _invoke(["evaluate", str(_task_dir(executed_dir))]) + regraded = _row(_task_dir(executed_dir)) + + assert regraded["final_status"] == graded["final_status"] + assert regraded["weighted_score"] == graded["weighted_score"] + assert regraded["max_turns_exhausted"] is True, "the fact must survive the grade too" def test_a_detached_grade_keeps_the_runs_api_routing_not_the_graders(tmp_path: Path) -> None: diff --git a/tests/test_experiment_reports.py b/tests/test_experiment_reports.py index 3e212da5..71f7d139 100644 --- a/tests/test_experiment_reports.py +++ b/tests/test_experiment_reports.py @@ -1708,3 +1708,102 @@ def test_experiment_report_snapshot_replicates(self): ) md = ExperimentReportGenerator.generate_experiment_report(result) assert_matches_snapshot(md, "experiment_replicates.md") + + +class TestUngradedRenderingInMarkdown: + """The Markdown twin of `TestUngradedRenderingInHtml`. + + Five ungraded edits landed on `reports_html.py` with tests and the identical + five on `reports_experiment.py` with none, so the two renderers could drift + apart while the suite stayed green. These also cover the four new public + `reports_stats` helpers, which no test called at all — including + `is_env_table_key`, a behaviour change (three key families newly hidden from + BOTH Environment tables) that shipped unasserted in either direction. + """ + + @staticmethod + def _ungraded_result(): + from coder_eval.models import ExperimentResult, VariantAggregate, VariantResult + from coder_eval.models.experiment import TaskExperimentSummary + + return ExperimentResult( + experiment_id="exec-only", + description="an execute run, not yet graded", + variant_ids=["a"], + task_summaries=[ + TaskExperimentSummary( + task_id="t", + variant_results=[ + VariantResult( + variant_id="a", + task_id="t", + weighted_score=None, + final_status="NOT_GRADED", + duration_seconds=12.0, + total_tokens=900, + ) + ], + best_variant="a", + is_tie=True, + score_spread=0.0, + ) + ], + variant_aggregates={ + "a": VariantAggregate( + variant_id="a", + tasks_run=1, + tasks_succeeded=0, + tasks_failed=0, + tasks_error=0, + tasks_not_graded=1, + average_score=None, + average_duration=12.0, + ) + }, + total_duration_seconds=12.0, + ) + + def test_the_report_names_the_fourth_bucket_and_publishes_no_zero(self): + from coder_eval.reports_experiment import ExperimentReportGenerator + + md = ExperimentReportGenerator.generate_experiment_report(self._ungraded_result()) + + assert "Not Graded" in md, "the fourth bucket must be named, or an execute run reads as 0 of 1" + # The score CELL, not the whole document — the spread column legitimately + # renders 0.000 for a single-variant experiment. + score_row = next(line for line in md.splitlines() if line.startswith("| t |")) + assert "n/a" in score_row, score_row + assert "0.000" not in score_row.split("|")[2], "an unmeasured row rendered as a scored zero" + + def test_duration_and_tokens_survive_an_ungraded_row(self): + """Only the SCORE is missing. Dropping the row whole made an all-ungraded + experiment render `Avg Duration | N/A` with Tokens absent entirely — the + bug `collect_variant_series`' own comment describes.""" + from coder_eval.reports_stats import collect_variant_series + + series = collect_variant_series(self._ungraded_result())["a"] + + assert series.scores == [] + assert series.durations == [12.0] + assert series.tokens == [900.0] + + def test_format_score_distinguishes_unmeasured_from_zero(self): + from coder_eval.reports_stats import UNGRADED_SCORE_TEXT, format_score + + assert format_score(None) == UNGRADED_SCORE_TEXT + assert format_score(0.0) == "0.000" + assert format_score(None) != format_score(0.0) + + @pytest.mark.parametrize( + "key", ["installed_tools", "command_base_path", "reference_digest", "graded_by_api_routing"] + ) + def test_env_table_hides_the_noise_keys(self, key): + from coder_eval.reports_stats import is_env_table_key + + assert not is_env_table_key(key) + + def test_env_table_keeps_ordinary_keys(self): + from coder_eval.reports_stats import is_env_table_key + + assert is_env_table_key("coder_eval") + assert is_env_table_key("api_routing") diff --git a/tests/test_reports.py b/tests/test_reports.py index 17253769..1dc85ae2 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -1307,3 +1307,60 @@ def test_generate_markdown_renders_both_markers_when_both_fire(): assert "max_turns exhausted" in report_md assert "expected_turns exceeded" in report_md assert "7/5" in report_md + + +class TestAnUnmeasuredRunPublishesNoRate: + """`tasks_graded` is the complement of the ungraded bucket, so it keeps ERROR + rows — right under `run`, where an errored row was attempted and genuinely + missed. Under `coder-eval execute` no criterion runs on ANY row, yet a + crashed one lands in the `error` bucket, so it stayed in the denominator on + its own: a 100-task execute night with 5 crashes published `pass_rate 0.0` + and `error_share 1.0`, a measured-looking total failure for a run that was + never measured at all. + """ + + @staticmethod + def _summary(*, succeeded: int, failed: int, error: int, not_graded: int) -> RunSummary: + return RunSummary( + run_id="2026-01-01_00-00-00", + start_time=datetime(2026, 1, 1), + end_time=datetime(2026, 1, 1), + total_duration_seconds=0.0, + tasks_run=succeeded + failed + error + not_graded, + tasks_succeeded=succeeded, + tasks_failed=failed, + tasks_error=error, + tasks_not_graded=not_graded, + task_results=[], + framework_version="0.1.0", + environment_info={}, + ) + + def test_an_execute_night_with_a_crash_reports_no_rate(self): + summary = self._summary(succeeded=0, failed=0, error=5, not_graded=95) + + assert summary.pass_rate is None + assert summary.error_share is None + + def test_the_markdown_says_so_rather_than_rendering_zero_percent(self): + summary = self._summary(succeeded=0, failed=0, error=5, not_graded=95) + + md = ReportGenerator.generate_markdown(summary) + + assert "n/a" in md + assert "0.0%" not in md, "a never-measured run rendered a measured-looking 0.0% pass rate" + + def test_a_partly_graded_run_still_reports_a_rate(self): + """The carve-out is only for "nothing was measured". Once ANY row reached + a verdict, an errored row is a genuine miss and belongs in the + denominator — the erroring-bonus this rule exists to prevent.""" + summary = self._summary(succeeded=3, failed=1, error=1, not_graded=5) + + assert summary.pass_rate == 3 / 5 + assert summary.error_share == 1 / 5 + + def test_an_ordinary_graded_run_is_untouched(self): + summary = self._summary(succeeded=2, failed=1, error=1, not_graded=0) + + assert summary.pass_rate == 0.5 + assert summary.error_share == 0.25 diff --git a/tests/test_resume.py b/tests/test_resume.py index 21282154..54bf8bbe 100644 --- a/tests/test_resume.py +++ b/tests/test_resume.py @@ -56,9 +56,19 @@ def _resolved(run_root, task_id: str) -> ResolvedTask: ) -def _write_task_json(rt: ResolvedTask, status: FinalStatus) -> None: - """Write a finalized task.json into rt.run_dir, like the orchestrator does.""" +def _write_task_json(rt: ResolvedTask, status: FinalStatus, *, graded: bool | None = None) -> None: + """Write a finalized task.json into rt.run_dir, like the orchestrator does. + + ``graded`` decides whether the row carries a verdict, and it is not the same + question as ``status``. That is the whole point of the resume partition: a + row is owed a grade when it was executed but never scored, and an ``execute`` + row that also tripped a run limit lands with an execution-fact status AND no + score. Defaulting it from the status alone would make the fixture unable to + express the case the partition exists to route. + """ rt.run_dir.mkdir(parents=True, exist_ok=True) + if graded is None: + graded = status.category != "ungraded" result = EvaluationResult( task_id=rt.task.task_id, task_description=rt.task.description, @@ -66,7 +76,9 @@ def _write_task_json(rt: ResolvedTask, status: FinalStatus) -> None: agent_type=AgentKind.CLAUDE_CODE, started_at=datetime.now(), final_status=status, - weighted_score=1.0 if status == FinalStatus.SUCCESS else 0.0, + # None, never 0.0, when nothing was measured: a laundered zero here would + # make the fixture indistinguishable from a genuinely-scored miss. + weighted_score=(1.0 if status == FinalStatus.SUCCESS else 0.0) if graded else None, duration_seconds=12.5, iteration_count=1, environment_info={}, @@ -125,15 +137,40 @@ def test_partition_treats_ungraded_as_done_for_execute(tmp_path): assert [tr.task_id for tr in part.prior_results] == ["ungraded_task"] +@pytest.mark.parametrize( + "status", + [FinalStatus.TIMEOUT, FinalStatus.TOKEN_BUDGET_EXCEEDED, FinalStatus.COST_BUDGET_EXCEEDED], +) +def test_an_execute_row_that_also_tripped_a_run_limit_still_owes_a_grade(tmp_path, status): + """The case routing-by-category missed entirely. + + A run limit aborts the run BEFORE grading, so under `coder-eval execute` such + a row lands with an execution-fact status — category `error`/`failed`, not + `ungraded` — and no verdict at all. Keying the partition on the category + filed it under "already complete", so it stayed permanently `weighted_score: + null` with an empty criteria vector in run.json, the rollup and the + evalboard — while `evaluate ` graded the identical bytes happily. + Two entry points into one feature disagreeing about the same record. + """ + row = _resolved(tmp_path, "limited_task") + _write_task_json(row, status, graded=False) + + part = partition_for_resume([row], grade=True) + + assert [rt.task.task_id for rt in part.to_grade] == ["limited_task"] + assert part.to_run == [], "the agent already ran; re-running it discards that spend" + + @pytest.mark.parametrize("status", [FinalStatus.FAILURE, FinalStatus.ERROR, FinalStatus.TIMEOUT]) def test_partition_still_never_retries_failures(tmp_path, status): - """The NOT_GRADED carve-out must not become a general 'retry bad rows' rule. + """The carve-out must not become a general 'retry bad rows' rule. Resume has never retried failures (delete a task's task.json to force that), - and both commands must keep treating them as complete. + and both commands must keep treating a row that CARRIES a verdict as + complete — which is why these rows are written graded. """ failed = _resolved(tmp_path, "failed_task") - _write_task_json(failed, status) + _write_task_json(failed, status, graded=True) for grade in (True, False): part = partition_for_resume([failed], grade=grade) diff --git a/tests/test_seed_from_prior_result.py b/tests/test_seed_from_prior_result.py index 10b602cc..6d88c05e 100644 --- a/tests/test_seed_from_prior_result.py +++ b/tests/test_seed_from_prior_result.py @@ -29,7 +29,9 @@ FileExistsCriterion, FinalStatus, PostRunResult, + SimulationTelemetry, TaskDefinition, + TurnRecord, parse_agent_config, ) from coder_eval.orchestrator import Orchestrator @@ -96,6 +98,14 @@ def _prior() -> EvaluationResult: started_at=datetime(2020, 1, 1, 0, 0, 0), final_status=FinalStatus.NOT_GRADED, iteration_count=7, + # Distinctive, NOT the model default. `iterations` and `simulation` were + # left at `[]` / `None`, so their parametrized sensors asserted `[] == []` + # and `None == None` — verified by deleting the `simulation` carry line in + # orchestrator.py and getting a byte-identical green suite, i.e. nothing + # anywhere guarded it and a re-graded simulation run would silently lose + # its dialog record. + iterations=[TurnRecord(iteration=1, user_input="prior prompt", agent_output="prior reply")], + simulation=SimulationTelemetry(n_trials=3, replicate_index=2, stop_reason="stop_token", total_turns=4), max_turns_exhausted=True, error_message="prior message", error_details={"where": "prior"}, @@ -161,6 +171,18 @@ def test_field_partition_covers_every_evaluation_result_field() -> None: def test_every_carried_field_reaches_the_regrade(field: str, tmp_path: Path) -> None: orch, prior = _seeded(tmp_path) assert orch.result is not None + # Anti-vacuity FIRST. This module's docstring claims every carried field is + # set to a distinctive value, and for two of sixteen it was not — so the + # assertion below compared a default against itself and would have passed + # with the production carry line deleted. Checking the fixture differs from + # the model default makes that failure mode impossible to reintroduce + # silently: a new CARRIED entry left at its default fails HERE, naming the + # fixture, rather than passing and pinning nothing. + default = EvaluationResult.model_fields[field].get_default(call_default_factory=True) + assert getattr(prior, field) != default, ( + f"_prior() leaves `{field}` at its model default, so the comparison below is vacuous — " + "it would pass with the carry line deleted. Give the fixture a distinctive value." + ) assert getattr(orch.result, field) == getattr(prior, field), ( f"_seed_from_prior_result dropped `{field}`; the graded row would report its default " "instead of what the run actually did." From 1652f406d3220a3e6a532522dd48a5501deeeaa6 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Fri, 4 Sep 2026 13:42:11 -0700 Subject: [PATCH 10/11] fix(security): close the three CodeQL findings on the detached-grading diff - write_text_atomic created its temp file 0o666, relying on the umask to reduce it. Under umask 0 that is a world-WRITABLE run record. Readable is the requirement (the host reads a container-written task.json back across the bind mount); writable never was. 0o644 keeps the fix and cannot widen. - The sandbox-escape guard logged a RESOLVED absolute path, which CodeQL reads as clear-text sensitive data. It was also the wrong string and the wrong place: the resolved path is just the author's own pattern joined onto a tempdir, and reporting from _within_sandbox fired once per rejected glob match. resolve_files now reports ONCE per criterion, naming the pattern the task author actually wrote. - Dropped a redundant local `import re` in tests/test_custom_lint.py. Co-Authored-By: Claude Opus 5 (1M context) --- src/coder_eval/path_utils.py | 7 +++--- src/coder_eval/sandbox.py | 28 ++++++++++++++++++----- tests/test_custom_lint.py | 2 -- tests/test_detached_grading_boundaries.py | 26 +++++++++++++++++++++ 4 files changed, 52 insertions(+), 11 deletions(-) diff --git a/src/coder_eval/path_utils.py b/src/coder_eval/path_utils.py index 7f68318c..1215b3fd 100644 --- a/src/coder_eval/path_utils.py +++ b/src/coder_eval/path_utils.py @@ -69,8 +69,9 @@ def write_text_atomic(path: Path, text: str) -> None: that is strictly better than wedging finalization, and the litter is recognisable by its embedded pid. - Mode is ``0o666`` so the umask applies, giving the same 0644 a plain - ``write_text`` produced. Creating it 0600 broke the docker driver on Linux: + Mode is ``0o644`` — the same mode a plain ``write_text`` produced, and the + widest one that is never group- or world-*writable* whatever the umask. + Creating it 0600 broke the docker driver on Linux: the in-container orchestrator writes ``task.json`` as root straight into the bind-mounted host run dir, and the host then reads it back as the invoking uid — an unguarded read that raises ``PermissionError`` for every task. A @@ -81,7 +82,7 @@ def write_text_atomic(path: Path, text: str) -> None: # predecessor, so O_EXCL can never collide with our own leftovers. tmp = path.with_name(f"{path.name}.{os.getpid()}.{secrets.token_hex(4)}.tmp") flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0) - fd = os.open(tmp, flags, 0o666) + fd = os.open(tmp, flags, 0o644) try: with os.fdopen(fd, "w", encoding="utf-8") as fh: fh.write(text) diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 06127eda..a19f6bc8 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -1315,6 +1315,12 @@ def _within_sandbox(self, candidate: Path) -> bool: indistinguishable to the criterion from a file that is not there, which is the same answer the template and mock-dir paths give, and raising here would book a config error as an agent crash (CE039). + + Silent by design — :meth:`resolve_files` reports the escape ONCE per + criterion, naming the pattern the task author actually wrote. Logging + here instead named a resolved absolute path (uninformative: it is the + author's own string joined onto a tempdir) once per rejected glob + match, so a wide pattern produced a burst of near-identical warnings. """ assert self.sandbox_dir is not None root = self.sandbox_dir.resolve() @@ -1322,12 +1328,13 @@ def _within_sandbox(self, candidate: Path) -> bool: resolved = candidate.resolve() except OSError: return False - if resolved == root or root in resolved.parents: - return True + return resolved == root or root in resolved.parents + + def _warn_escaped(self, path: str) -> None: + """Report that a criterion's declared path left the sandbox.""" logger.warning( - "Criterion path %r resolves outside the sandbox (%s); treating it as no match.", str(candidate), root + "Criterion path %r resolves outside the sandbox (%s); treating it as no match.", path, self.sandbox_dir ) - return False def resolve_files(self, path: str) -> list[Path]: """Resolve a criterion ``path`` to the sandbox files it addresses. @@ -1363,7 +1370,10 @@ def resolve_files(self, path: str) -> list[Path]: # Literal first: an existing path is never reinterpreted as a pattern. candidate = self.sandbox_dir / path if candidate.exists(): - return [candidate] if self._within_sandbox(candidate) else [] + if self._within_sandbox(candidate): + return [candidate] + self._warn_escaped(path) + return [] if not _is_glob(path): return [] @@ -1372,14 +1382,20 @@ def resolve_files(self, path: str) -> list[Path]: pinned = {segment for segment in path.split("/") if segment and not _is_glob(segment)} matches: list[Path] = [] + escaped = False for match in self.sandbox_dir.glob(path): - if not match.is_file() or not self._within_sandbox(match): + if not match.is_file(): + continue + if not self._within_sandbox(match): + escaped = True continue discovered = [part for part in match.relative_to(self.sandbox_dir).parts if part not in pinned] if discovered and should_ignore_path(Path(*discovered), patterns): continue matches.append(match) + if escaped: + self._warn_escaped(path) return sorted(matches) def resolved_path_label(self, path: str) -> str | None: diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 900a7ff1..4ebb8bc4 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -3847,8 +3847,6 @@ def test_every_registered_rule_is_listed(self): def test_every_listed_id_is_well_formed(self): """Cheap guard against a typo silently widening the allowlist.""" - import re - bad = sorted(i for i in self._external() if not re.fullmatch(r"CE\d{3}", i)) assert not bad, f"not a CE rule id: {bad}" diff --git a/tests/test_detached_grading_boundaries.py b/tests/test_detached_grading_boundaries.py index 922aa553..79a9734a 100644 --- a/tests/test_detached_grading_boundaries.py +++ b/tests/test_detached_grading_boundaries.py @@ -14,6 +14,7 @@ from __future__ import annotations import json +import logging import os from datetime import datetime from pathlib import Path @@ -378,6 +379,10 @@ def test_the_record_is_readable_by_other_uids(self, tmp_path: Path) -> None: mode = target.stat().st_mode & 0o777 assert mode & 0o044, f"task.json is not group/other-readable: {mode:#o}" + # Readable is the requirement; WRITABLE is not. A create mode of 0o666 + # produced 0644 under the usual umask and a world-writable record under + # umask 0 — the widest mode that keeps the fix is 0o644. + assert not mode & 0o022, f"task.json is group/other-writable: {mode:#o}" def test_an_ordinary_write_still_works(self, tmp_path: Path) -> None: from coder_eval.path_utils import write_text_atomic @@ -672,3 +677,24 @@ def test_an_ordinary_in_sandbox_path_still_resolves(self, tmp_path: Path) -> Non assert sandbox.resolve_files("proof.txt") == [work / "proof.txt"] assert sandbox.resolve_files("*.txt") == [work / "proof.txt"] + + def test_the_escape_is_reported_once_naming_the_authored_pattern( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """One warning per criterion, naming the string the task author wrote. + + Reporting from `_within_sandbox` instead fired once per rejected glob + match — a burst of near-identical lines for a wide pattern — and named + a resolved absolute path, which is just that same string joined onto a + tempdir. + """ + sandbox, _ = self._sandbox(tmp_path) + for name in ("a.txt", "b.txt", "c.txt"): + (tmp_path / name).write_text("token", encoding="utf-8") + + with caplog.at_level(logging.WARNING, logger="coder_eval.sandbox"): + assert sandbox.resolve_files("../*.txt") == [] + + escapes = [r for r in caplog.records if "resolves outside the sandbox" in r.getMessage()] + assert len(escapes) == 1, [r.getMessage() for r in escapes] + assert "'../*.txt'" in escapes[0].getMessage() From 9e479ae0b908e9edc87741b1f87971acb7969c45 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Fri, 4 Sep 2026 15:32:44 -0700 Subject: [PATCH 11/11] fix(tests): kill the CodeQL taint source and the Windows mode assertion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both new findings trace to the SAME file, and neither is a production bug. - The high-severity py/clear-text-logging-sensitive-data alert against sandbox.py had its taint SOURCE in this test: a local named `secret`. CodeQL's sensitive-data heuristic keys on the identifier, so the fixture flowed through resolve_files into the sandbox-escape warning and indicted production code that only logs a task-authored glob pattern. Renamed to `outside` — which is also the more accurate name, since what the fixture stands for is a file OUTSIDE the sandbox — with the reason recorded on the class so nobody renames it back. No lint rule for this: the pattern is "a test identifier a scanner's heuristic reads as sensitive", which cannot be detected without reimplementing that heuristic, and a name denylist over tests/ would be loud and wrong far more often than right. - The mode assertion added last round fails on Windows, which has no POSIX mode bits: os.stat reports 0o666 for any writable file whatever the create mode. The assertion is about a docker-on-Linux bind mount, so skip it there rather than weaken it everywhere. Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_detached_grading_boundaries.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/test_detached_grading_boundaries.py b/tests/test_detached_grading_boundaries.py index 79a9734a..468ae91e 100644 --- a/tests/test_detached_grading_boundaries.py +++ b/tests/test_detached_grading_boundaries.py @@ -377,6 +377,9 @@ def test_the_record_is_readable_by_other_uids(self, tmp_path: Path) -> None: target = tmp_path / "task.json" write_text_atomic(target, "hello") + if os.name == "nt": + pytest.skip("Windows has no POSIX mode bits; stat reports 0o666 for any writable file") + mode = target.stat().st_mode & 0o777 assert mode & 0o044, f"task.json is not group/other-readable: {mode:#o}" # Readable is the requirement; WRITABLE is not. A create mode of 0o666 @@ -639,6 +642,12 @@ class TestCriterionPathsCannotEscapeTheSandbox: `evaluate ` began rebuilding the criteria list from a shareable run directory, which turns `file_contains` / `file_check` / `file_matches_regex` into a pass-fail oracle over any file the grading user can read. + + The out-of-sandbox fixture is called ``outside``, NOT ``secret``: CodeQL's + sensitive-data heuristic keys on the identifier, so a local named ``secret`` + became a taint SOURCE, flowed through ``resolve_files`` into the escape + warning, and raised a high-severity `py/clear-text-logging-sensitive-data` + against production code that only logs a task-authored glob pattern. """ @staticmethod @@ -653,20 +662,20 @@ def _sandbox(tmp_path: Path): def test_an_absolute_path_resolves_to_nothing(self, tmp_path: Path) -> None: sandbox, _ = self._sandbox(tmp_path) - secret = tmp_path / "secret.txt" - secret.write_text("token", encoding="utf-8") + outside = tmp_path / "outside.txt" + outside.write_text("payload", encoding="utf-8") - assert sandbox.resolve_files(str(secret)) == [] + assert sandbox.resolve_files(str(outside)) == [] def test_a_dotdot_traversal_resolves_to_nothing(self, tmp_path: Path) -> None: sandbox, _ = self._sandbox(tmp_path) - (tmp_path / "secret.txt").write_text("token", encoding="utf-8") + (tmp_path / "outside.txt").write_text("payload", encoding="utf-8") - assert sandbox.resolve_files("../secret.txt") == [] + assert sandbox.resolve_files("../outside.txt") == [] def test_a_glob_cannot_escape_either(self, tmp_path: Path) -> None: sandbox, _ = self._sandbox(tmp_path) - (tmp_path / "secret.txt").write_text("token", encoding="utf-8") + (tmp_path / "outside.txt").write_text("payload", encoding="utf-8") assert sandbox.resolve_files("../*.txt") == []