perf(agent): accelerate local verification scheduling and reporting (#460)

* perf(agent): accelerate local verification scheduling and reporting

* perf(agent): shard the security spec, widen the UI test pool, open SSE streams at once

Per-task timings showed the security spec as the critical path: one bun
process for 22 files, 100 s, while everything else finished by 56 s.

- Security shards: locally the spec files are packed into as many shards
  as browser workers (four on a large host), each on its own lane database
  and Valkey index (app_security_1..4); an aggregate task merges the JUnit
  reports so evidence and the findings manifest read one run. Packing uses
  per-file durations recorded from the previous run's report (size only
  breaks ties), so a cold checkout balances on its second run. With one
  worker the spec runs whole on app_security exactly as before.
- UI pool: vitest runs on half the slot budget, capped at eight, once the
  spec is no longer the long pole (56 s -> 38 s).
- The SSE stream now yields a ping on open. Elysia turns a generator into a
  response only after its first yield, so a silent stream kept the response
  headers, and a browser's EventSource open event, waiting for a
  notification or the 25 s keepalive; the spec's HTTP tests spent 25 s each
  on exactly that. The generator fixture consumes the handshake.

Release-local on a 20-core host: 104 s -> 49 s (three passing runs: 50,
50, 49). The SSE spec: 79 s -> 6 s.

* perf(agent): shard the API test suite too, with coverage merged from the shards

api.tests was the next critical path at ~40 s in one process. It now
shards like the security spec (app_tests_1..4, Valkey 9..12), packed by
recorded per-file durations, with an aggregate task that merges the JUnit
reports for inventory evidence.

Coverage in release-local comes from the shards' LCOV reports. Bun's
"All files" row is the unweighted mean of per-file percentages (verified
against bun test --coverage on the same run), and that is what the merge
reproduces: lines union exactly per file; functions can only be combined
as the best shard per file because Bun's LCOV has no per-function records,
so when that lower bound alone misses the floor the whole suite runs once
through check-coverage.ts and its verdict decides. The floor constants
move to coverage-thresholds.ts, shared by both paths.

Release-local on a 20-core host: 44 s, twice in a row (from 49 s; from
104 s before sharding; from ~10 min before parallel lanes). The remaining
floor is ui.tests at ~40 s.

Author: agjs

apps/api/scripts/quality/check-coverage.ts

@@ -5,14 +5,12 @@
  * `coverageThreshold` is documented but not enforced, so we parse the
  * text report ourselves and exit non-zero on regression.
  *
- * The threshold is a ratchet, not a wishlist: it sits a few points
- * below the current measured rate so a small slip triggers the alarm.
- * Raise it as coverage climbs; never lower it to silence a regression.
+ * The floor lives in coverage-thresholds.ts, shared with the sharded
+ * verification runner.
  */
 import { spawnSync } from "node:child_process";
+import { MIN_FUNCTION, MIN_LINE } from "./coverage-thresholds";
 
-const MIN_LINE = 0.65;
-const MIN_FUNCTION = 0.7;
 const MAX_TEST_OUTPUT_BUFFER_BYTES = 64 * 1024 * 1024;
 
 const FORBIDDEN_OUTPUT = [
@@ -49,16 +47,26 @@ const runCoverage = (): {
    * letting them into the coverage gate turns the ordinary merge gate red
    * for reasons unrelated to the change under review.
    */
-  const result = spawnSync("bun", ["test", "tests", "--coverage"], {
-    encoding: "utf8",
-    maxBuffer: MAX_TEST_OUTPUT_BUFFER_BYTES,
-    env: {
-      ...process.env,
-      NODE_ENV: "test",
-      LOG_LEVEL: "error",
-      NODE_NO_WARNINGS: "1",
-    },
-  });
+  const result = spawnSync(
+    "bun",
+    [
+      ...(process.env.AGENT_SANDBOX === "1" ? ["--no-env-file"] : []),
+      "test",
+      "tests",
+      "--coverage",
+      ...process.argv.slice(2),
+    ],
+    {
+      encoding: "utf8",
+      maxBuffer: MAX_TEST_OUTPUT_BUFFER_BYTES,
+      env: {
+        ...process.env,
+        NODE_ENV: "test",
+        LOG_LEVEL: "error",
+        NODE_NO_WARNINGS: "1",
+      },
+    }
+  );
 
   return {
     combined: result.stdout + result.stderr,
@@ -80,7 +88,14 @@ const parseAllFilesRow = (output: string): ICoverageResult | null => {
   const functionPct = parseFloat(parts[1] ?? "");
   const linePct = parseFloat(parts[2] ?? "");
 
-  if (Number.isNaN(linePct) || Number.isNaN(functionPct)) {
+  if (
+    !Number.isFinite(linePct) ||
+    !Number.isFinite(functionPct) ||
+    linePct < 0 ||
+    linePct > 100 ||
+    functionPct < 0 ||
+    functionPct > 100
+  ) {
     return null;
   }
 
@@ -118,7 +133,7 @@ if (warningLines.length > 0) {
     console.error(`  ${line}`);
   }
 
-  process.exit(1);
+  process.exit(86);
 }
 
 if (exitCode !== 0) {
@@ -145,7 +160,7 @@ if (!lineOk || !functionOk) {
       `\n\nTo raise: add tests for under-covered surfaces (queues / SSE / web push / setup).` +
       `\nTo lower the threshold: do not. Treat the gate as a ratchet.`
   );
-  process.exit(1);
+  process.exit(86);
 }
 
 console.log(

apps/api/scripts/quality/coverage-thresholds.ts

@@ -0,0 +1,9 @@
+/*
+ * The coverage floor is a ratchet, not a wishlist: it sits a few points
+ * below the current measured rate so a small slip triggers the alarm.
+ * Raise it as coverage climbs; never lower it to silence a regression.
+ * Shared by the single-process gate (check-coverage.ts) and the sharded
+ * verification runner, so both enforce the same numbers.
+ */
+export const MIN_LINE = 0.65;
+export const MIN_FUNCTION = 0.7;

apps/api/security-spec/f14-sse-stream-lifetime.test.ts

@@ -129,8 +129,32 @@ const openStream = (userId: string, jti: string): IStreamFixture => {
     }),
   });
 
+  /*
+   * The stream opens with a ping so Elysia can flush the response headers
+   * before the first notification. The fixture consumes it, so every test
+   * reads the real payloads exactly as it would have without the handshake.
+   */
+  let opened = false;
+
+  const next = async (): Promise<IteratorResult<string, void>> => {
+    if (!opened) {
+      opened = true;
+
+      const handshake = await generator.next();
+
+      if (
+        handshake.done === true ||
+        handshake.value !== JSON.stringify({ type: "ping" })
+      ) {
+        throw new Error("f14: the stream did not open with a ping");
+      }
+    }
+
+    return generator.next();
+  };
+
   return {
-    next: () => generator.next(),
+    next,
     publish: (message) =>
       valkeyPubSub.publish(userNotificationChannel(userId), message),
     credential,

apps/api/src/api/notifications/notifications.sse.ts

@@ -157,6 +157,15 @@ export const notificationsStreamHandler = async function* (
   let lastPingAtMs = nowMs();
 
   try {
+    /*
+     * Elysia turns a generator into a response only after its first
+     * `yield`. A stream that stays silent until a notification arrives, or
+     * until the keepalive below, would keep the browser's `EventSource`
+     * from opening and any proxy from seeing bytes for up to 25 seconds.
+     * A ping on open flushes the headers at once; the client ignores it.
+     */
+    yield JSON.stringify({ type: "ping" });
+
     while (!isAborted()) {
       /*
        * The credential is re-checked before every payload, not once per

apps/api/src/templates/email/build.ts

@@ -103,6 +103,18 @@ const precompilePartials = (): Record<string, string> => {
   return partials;
 };
 
+/** Avoid replacing artifacts while parallel test and build consumers read them. */
+const writeArtifact = (outputPath: string, content: string): void => {
+  if (
+    fs.existsSync(outputPath) &&
+    fs.readFileSync(outputPath, "utf8") === content
+  ) {
+    return;
+  }
+
+  fs.writeFileSync(outputPath, content, "utf8");
+};
+
 const buildTemplate = (templatePath: string): void => {
   const source = fs.readFileSync(templatePath, "utf8");
   const baseTemplate = precompileToString(source);
@@ -123,10 +135,9 @@ const buildTemplate = (templatePath: string): void => {
     contentTemplate = precompileToString(contentSource);
   }
 
-  fs.writeFileSync(
+  writeArtifact(
     outputPath,
-    JSON.stringify({ baseTemplate, contentTemplate }, null, 2),
-    "utf8"
+    JSON.stringify({ baseTemplate, contentTemplate }, null, 2)
   );
   console.log(`✓ Built: ${path.relative(__dirname, outputPath)}`);
 };
@@ -137,7 +148,7 @@ const buildPartialsManifest = (): void => {
   fs.ensureDirSync(DIST_DIR);
   const manifestPath = path.join(DIST_DIR, "partials.json");
 
-  fs.writeFileSync(manifestPath, JSON.stringify(partials, null, 2), "utf8");
+  writeArtifact(manifestPath, JSON.stringify(partials, null, 2));
   console.log(
     `✓ Built partials manifest: ${path.relative(__dirname, manifestPath)}`
   );

apps/api/tsconfig.json

@@ -1,5 +1,7 @@
 {
   "compilerOptions": {
+    "incremental": true,
+    "tsBuildInfoFile": "./node_modules/.cache/tsc/api.tsbuildinfo",
     "target": "ES2022",
     "module": "ES2022",
     "moduleResolution": "bundler",
@@ -29,6 +31,11 @@
       "@/*": ["./src/*"]
     }
   },
-  "include": ["src/**/*.ts", "tests/**/*.ts", "security-spec/**/*.ts", "scripts/**/*.ts"],
+  "include": [
+    "src/**/*.ts",
+    "tests/**/*.ts",
+    "security-spec/**/*.ts",
+    "scripts/**/*.ts"
+  ],
   "exclude": ["node_modules", "dist", "drizzle", "src/templates/email/dist"]
 }

apps/ui/.size-limit.json

@@ -9,7 +9,8 @@
       "dist/assets/query-*.js"
     ],
     "limit": "255 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "Modulepreload runtime + shared shell",
@@ -24,120 +25,140 @@
       "dist/assets/dist-*.js"
     ],
     "limit": "165 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "CSS (Tailwind compiled)",
     "path": "dist/assets/*.css",
     "limit": "12 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "LoginPage chunk",
     "path": "dist/assets/LoginPage-*.js",
     "limit": "20 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "DashboardPage chunk",
     "path": "dist/assets/DashboardPage-*.js",
     "limit": "5 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "SettingsPage chunk",
     "path": "dist/assets/SettingsPage-*.js",
     "limit": "20 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "BillingPage chunk",
     "path": "dist/assets/BillingPage-*.js",
     "limit": "4 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "NotificationsPage chunk",
     "path": "dist/assets/NotificationsPage-*.js",
     "limit": "4 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "NotificationsPreferencesPage chunk",
     "path": "dist/assets/NotificationsPreferencesPage-*.js",
     "limit": "3 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "InvitationsPage chunk",
     "path": "dist/assets/InvitationsPage-*.js",
     "limit": "4 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "InvitationAcceptPage chunk",
     "path": "dist/assets/InvitationAcceptPage-*.js",
     "limit": "3 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "OwnershipTransferAcceptPage chunk",
     "path": "dist/assets/OwnershipTransferAcceptPage-*.js",
     "limit": "3 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "JoinRequestsPage chunk",
     "path": "dist/assets/JoinRequestsPage-*.js",
     "limit": "3 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "AuditLogPage chunk",
     "path": "dist/assets/AuditLogPage-*.js",
     "limit": "3 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "ProfilePage chunk",
     "path": "dist/assets/ProfilePage-*.js",
     "limit": "3 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "SignUpPage chunk",
     "path": "dist/assets/SignUpPage-*.js",
     "limit": "3 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "ForgotPasswordPage chunk",
     "path": "dist/assets/ForgotPasswordPage-*.js",
     "limit": "3 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "ResetPasswordPage chunk",
     "path": "dist/assets/ResetPasswordPage-*.js",
     "limit": "3 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "VerifyEmailPage chunk",
     "path": "dist/assets/VerifyEmailPage-*.js",
     "limit": "3 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "OAuthCallbackPage chunk",
     "path": "dist/assets/OAuthCallbackPage-*.js",
     "limit": "3 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   },
   {
     "name": "NotFoundPage chunk",
     "path": "dist/assets/NotFoundPage-*.js",
     "limit": "2 KB",
-    "gzip": true
+    "gzip": true,
+    "running": false
   }
 ]

apps/ui/scripts/codegen/new-feature.ts

@@ -322,7 +322,8 @@ if (namespaceEnabled) {
     name: `${Name} translations (all locales)`,
     path: `dist/assets/${lower}-*.js`,
     limit: "10 KB",
-    gzip: true
+    gzip: true,
+    running: false
   });
   writeFileSync(
     budgetPath,