1313`testsuite` by hand. tests/plans/testkit.yaml says which suites run in which
1414environments, and tests/environments/*.yaml say how each environment is
1515provisioned. This check derives the lane triples from the plan and fails when a
16- workflow matrix lists something different, so the two cannot drift apart.
16+ workflow matrix lists something different, so the two cannot drift apart. It also
17+ checks that each testsuite's `suite_expected_tests` in tests/tmachine/config.nix equals the
18+ number of tests the plan expects, using tests/expected-matrix.json.
1719
1820Each plan entry's suite maps to a tmachine testsuite name below.
1921"""
2022
2123from __future__ import annotations
2224
2325import json
26+ import re
2427import subprocess
2528import sys
2629from pathlib import Path
2730
2831import yaml
2932
3033PLAN = "tests/plans/testkit.yaml"
34+ CONFIG = "tests/tmachine/config.nix"
35+ EXPECTED = "tests/expected-matrix.json"
3136WORKFLOWS = (
3237 ".github/workflows/release-qualification.yml" ,
3338 ".github/workflows/release-dev.yml" ,
39+ ".github/workflows/branch-e2e.yml" ,
3440)
3541# (plan family, suite or None for the family default) -> tmachine testsuite.
3642TESTSUITES = {
@@ -60,7 +66,53 @@ def environments(base: Path) -> dict[str, tuple[str, str]]:
6066 return found
6167
6268
69+ def selects (testsuite : str , test : str ) -> bool :
70+ """Whether a catalog entry `family/suite/test` belongs to a testsuite."""
71+ family , suite , _ = test .split ("/" , 2 )
72+ if testsuite == "oci-image" :
73+ return (family , suite ) == ("sandbox" , "oci-image" )
74+ if testsuite == "sandbox" :
75+ return family == "sandbox" and suite != "oci-image"
76+ if testsuite == "provider-refresh" :
77+ return family == "providers"
78+ return False
79+
80+
81+ def configured_counts (base : Path ) -> dict [str , int ]:
82+ """`suite_expected_tests` for each testsuite in tests/tmachine/config.nix."""
83+ text = (base / CONFIG ).read_text ()
84+ counts = {}
85+ for match in re .finditer (r'name = "([^"]+)";' , text ):
86+ rest = text [match .end () :]
87+ end = re .search (r'\n \{' , rest )
88+ block = rest [: end .start ()] if end else rest
89+ count = re .search (r'suite_expected_tests = "(\d+)"' , block )
90+ if count :
91+ counts [match .group (1 )] = int (count .group (1 ))
92+ return counts
93+
94+
95+ def count_problems (base : Path ) -> list [str ]:
96+ """Each lane's `suite_expected_tests` must equal what the plan expects."""
97+ tests = json .loads ((base / EXPECTED ).read_text ())["plans" ]["testkit" ]["tests" ]
98+ configured = configured_counts (base )
99+ problems = []
100+ for (environment , installer , testsuite ), env_id in sorted (plan_lanes (base )):
101+ expected = sum (selects (testsuite , test ) for test in tests [env_id ])
102+ if configured .get (testsuite ) != expected :
103+ problems .append (
104+ f"error: { CONFIG } sets suite_expected_tests={ configured .get (testsuite )} for "
105+ f"{ testsuite } , but { PLAN } expects { expected } tests in { environment } "
106+ )
107+ return problems
108+
109+
63110def from_plan (base : Path ) -> set [tuple [str , str , str ]]:
111+ return {triple for triple , _ in plan_lanes (base )}
112+
113+
114+ def plan_lanes (base : Path ) -> set [tuple [tuple [str , str , str ], str ]]:
115+ """Each lane as ((environment, installer, testsuite), plan environment id)."""
64116 envs = environments (base )
65117 plan = yaml .safe_load ((base / PLAN ).read_text ())
66118 triples = set ()
@@ -75,7 +127,7 @@ def from_plan(base: Path) -> set[tuple[str, str, str]]:
75127 raise SystemExit (f"error: no tmachine testsuite for { family } { suites } " )
76128 for env_id in entry ["environments" ]:
77129 environment , installer = envs [env_id ]
78- triples .add ((environment , installer , name ))
130+ triples .add ((( environment , installer , name ), env_id ))
79131 return triples
80132
81133
@@ -107,7 +159,10 @@ def main() -> int:
107159 for triple in sorted (missing ):
108160 print (f"error: { workflow } is missing { triple } from { PLAN } " , file = sys .stderr )
109161 failed |= bool (extra or missing )
110- return 1 if failed else 0
162+ problems = count_problems (base )
163+ for problem in problems :
164+ print (problem , file = sys .stderr )
165+ return 1 if failed or problems else 0
111166
112167
113168if __name__ == "__main__" :
0 commit comments