Skip to content

Commit 34caea4

Browse files
ci(tests): check lane test counts and Branch E2E against the plan
The drift check now also covers Branch E2E, and compares each testsuite's `suite_expected_tests` in tests/tmachine/config.nix with the number of tests the plan expects, using tests/expected-matrix.json. Raising a count in one place without the other fails the check. Document why persistent_workspace_survives_stop_start doesn't declare `requires`: the tmachine playbooks don't pass capabilities yet. Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>
1 parent 116e0a9 commit 34caea4

2 files changed

Lines changed: 62 additions & 3 deletions

File tree

‎tasks/scripts/check_plan_workflow_matrix.py‎

Lines changed: 58 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,24 +13,30 @@
1313
`testsuite` by hand. tests/plans/testkit.yaml says which suites run in which
1414
environments, and tests/environments/*.yaml say how each environment is
1515
provisioned. This check derives the lane triples from the plan and fails when a
16-
workflow matrix lists something different, so the two cannot drift apart.
16+
workflow matrix lists something different, so the two cannot drift apart. It also
17+
checks that each testsuite's `suite_expected_tests` in tests/tmachine/config.nix equals the
18+
number of tests the plan expects, using tests/expected-matrix.json.
1719
1820
Each plan entry's suite maps to a tmachine testsuite name below.
1921
"""
2022

2123
from __future__ import annotations
2224

2325
import json
26+
import re
2427
import subprocess
2528
import sys
2629
from pathlib import Path
2730

2831
import yaml
2932

3033
PLAN = "tests/plans/testkit.yaml"
34+
CONFIG = "tests/tmachine/config.nix"
35+
EXPECTED = "tests/expected-matrix.json"
3136
WORKFLOWS = (
3237
".github/workflows/release-qualification.yml",
3338
".github/workflows/release-dev.yml",
39+
".github/workflows/branch-e2e.yml",
3440
)
3541
# (plan family, suite or None for the family default) -> tmachine testsuite.
3642
TESTSUITES = {
@@ -60,7 +66,53 @@ def environments(base: Path) -> dict[str, tuple[str, str]]:
6066
return found
6167

6268

69+
def selects(testsuite: str, test: str) -> bool:
70+
"""Whether a catalog entry `family/suite/test` belongs to a testsuite."""
71+
family, suite, _ = test.split("/", 2)
72+
if testsuite == "oci-image":
73+
return (family, suite) == ("sandbox", "oci-image")
74+
if testsuite == "sandbox":
75+
return family == "sandbox" and suite != "oci-image"
76+
if testsuite == "provider-refresh":
77+
return family == "providers"
78+
return False
79+
80+
81+
def configured_counts(base: Path) -> dict[str, int]:
82+
"""`suite_expected_tests` for each testsuite in tests/tmachine/config.nix."""
83+
text = (base / CONFIG).read_text()
84+
counts = {}
85+
for match in re.finditer(r'name = "([^"]+)";', text):
86+
rest = text[match.end() :]
87+
end = re.search(r'\n \{', rest)
88+
block = rest[: end.start()] if end else rest
89+
count = re.search(r'suite_expected_tests = "(\d+)"', block)
90+
if count:
91+
counts[match.group(1)] = int(count.group(1))
92+
return counts
93+
94+
95+
def count_problems(base: Path) -> list[str]:
96+
"""Each lane's `suite_expected_tests` must equal what the plan expects."""
97+
tests = json.loads((base / EXPECTED).read_text())["plans"]["testkit"]["tests"]
98+
configured = configured_counts(base)
99+
problems = []
100+
for (environment, installer, testsuite), env_id in sorted(plan_lanes(base)):
101+
expected = sum(selects(testsuite, test) for test in tests[env_id])
102+
if configured.get(testsuite) != expected:
103+
problems.append(
104+
f"error: {CONFIG} sets suite_expected_tests={configured.get(testsuite)} for "
105+
f"{testsuite}, but {PLAN} expects {expected} tests in {environment}"
106+
)
107+
return problems
108+
109+
63110
def from_plan(base: Path) -> set[tuple[str, str, str]]:
111+
return {triple for triple, _ in plan_lanes(base)}
112+
113+
114+
def plan_lanes(base: Path) -> set[tuple[tuple[str, str, str], str]]:
115+
"""Each lane as ((environment, installer, testsuite), plan environment id)."""
64116
envs = environments(base)
65117
plan = yaml.safe_load((base / PLAN).read_text())
66118
triples = set()
@@ -75,7 +127,7 @@ def from_plan(base: Path) -> set[tuple[str, str, str]]:
75127
raise SystemExit(f"error: no tmachine testsuite for {family} {suites}")
76128
for env_id in entry["environments"]:
77129
environment, installer = envs[env_id]
78-
triples.add((environment, installer, name))
130+
triples.add(((environment, installer, name), env_id))
79131
return triples
80132

81133

@@ -107,7 +159,10 @@ def main() -> int:
107159
for triple in sorted(missing):
108160
print(f"error: {workflow} is missing {triple} from {PLAN}", file=sys.stderr)
109161
failed |= bool(extra or missing)
110-
return 1 if failed else 0
162+
problems = count_problems(base)
163+
for problem in problems:
164+
print(problem, file=sys.stderr)
165+
return 1 if failed or problems else 0
111166

112167

113168
if __name__ == "__main__":

‎tests/suites/sandbox/README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ leave driver behavior alone.
4242
- If a behavior is left to the driver, declare it with
4343
`#[testkit::test(requires = [...])]`. The runner supplies the environment's
4444
capabilities, and the test fails if one is missing.
45+
The tmachine playbooks and `mise run e2e:testkit` don't supply capabilities
46+
yet, so a test with `requires` fails there. That is why
47+
`persistent_workspace_survives_stop_start` doesn't declare
48+
`persistent_workspace`; add it when the runner drives those lanes.
4549
- Use the CLI for upload, download, connect and forward. Rely on its exit code
4650
and `--output json`.
4751
- Add a testkit helper instead of calling raw gRPC from a test.

0 commit comments

Comments
 (0)