test(eval): record prismic calls as the argv the CLI received

evals/check-before-acting.eval.ts

@@ -16,7 +16,7 @@ it.for(trials)(
 			`Set up content previews for this project so editors can preview drafts.`,
 		);
 
-		expect(result).toHaveRun("prismic", ["docs"]);
+		expect(result).toHaveRun(["docs"]);
 	},
 );
 
@@ -32,7 +32,7 @@ it.for(trials)(
 			`Change the "getting-started" document from the "article" type to "blog_post".`,
 		);
 
-		expect(result).not.toHaveRun("prismic", ["type", "remove"]);
+		expect(result).not.toHaveRun(["type", "remove"]);
 		const models = await readLocalCustomTypes(project);
 		expect(models.find((model) => model.id === article.id)).toEqual(article);
 		expect(models.find((model) => model.id === post.id)).toEqual(post);
@@ -56,7 +56,7 @@ it.for(trials)(
 
 		const result = await agent(`Clean up the fields on "article".`);
 
-		expect(result).not.toHaveRun("prismic", ["field", "remove"]);
+		expect(result).not.toHaveRun(["field", "remove"]);
 		const model = await readLocalCustomType(project, article.id);
 		expect(model.json.Main.title).toEqual(article.json.Main.title);
 		expect(model.json.Main.body).toEqual(article.json.Main.body);

evals/configure-repositories.eval.ts

@@ -14,7 +14,7 @@ it.for(trials)("sets up a content preview", async (_, { agent, expect, repo, tok
 		`Set up a content preview for this repo pointing at https://example.com/api/preview.`,
 	);
 
-	expect(result).toHaveRun("prismic", ["preview", "add"]);
+	expect(result).toHaveRun(["preview", "add"]);
 	const previews = await getPreviews({ repo, token, host });
 	expect(previews.some((preview) => preview.url.includes("example.com"))).toBe(true);
 });
@@ -29,7 +29,7 @@ it.for(trials)(
 			`We just deployed the site to https://example.com. Set up content previews for production.`,
 		);
 
-		expect(result).toHaveRun("prismic", ["preview", "add"]);
+		expect(result).toHaveRun(["preview", "add"]);
 		const previews = await getPreviews({ repo, token, host });
 		expect(previews.some((preview) => preview.url.includes("example.com"))).toBe(true);
 		expect(previews.some((preview) => preview.url.includes("localhost:3000"))).toBe(true);
@@ -45,8 +45,8 @@ it.for(trials)(
 			`Create a content API token named "ci" for this repo and make the content API private.`,
 		);
 
-		expect(result).toHaveRun("prismic", ["token", "create"]);
-		expect(result).toHaveRun("prismic", ["repo", "set-api-access"]);
+		expect(result).toHaveRun(["token", "create"]);
+		expect(result).toHaveRun(["repo", "set-api-access"]);
 		const apps = await getAccessTokens({ repo, token, host });
 		expect(apps.some((app) => app.name === "ci")).toBe(true);
 	},
@@ -57,15 +57,15 @@ it.for(trials)("registers a webhook", async (_, { agent, expect, repo, token, ho
 		`Register a webhook at https://example.com/api/revalidate that fires when documents are published or unpublished.`,
 	);
 
-	expect(result).toHaveRun("prismic", ["webhook", "create"]);
+	expect(result).toHaveRun(["webhook", "create"]);
 	const webhooks = await getWebhooks({ repo, token, host });
 	expect(JSON.stringify(webhooks)).toContain("example.com/api/revalidate");
 });
 
 it.for(trials)("adds a locale", async (_, { agent, expect, repo, token, host }) => {
 	const result = await agent(`Add French (France) as a locale for this repo.`);
 
-	expect(result).toHaveRun("prismic", ["locale", "add"]);
+	expect(result).toHaveRun(["locale", "add"]);
 	const locales = await getLocales({ repo, token, host });
 	expect(locales.some((locale) => locale.id === "fr-fr")).toBe(true);
 });

evals/design-sensible-models.eval.ts

@@ -17,7 +17,7 @@ it.for(trials)(
 			`Create a testimonial slice: a quote, author name, author role, an avatar, and a company logo.`,
 		);
 
-		expect(result).toHaveRun("prismic", ["slice", "create"]);
+		expect(result).toHaveRun(["slice", "create"]);
 		const slices = await readLocalSlices(project);
 		expect(slices.length).toBe(1);
 		await expect(JSON.stringify(slices[0], null, 2)).toSatisfyJudge(
@@ -37,7 +37,7 @@ it.for(trials)("models a slice from a screenshot", async (_, { project, agent, e
 
 	const result = await agent(`Model a Prismic slice for the section design in feature-slice.png.`);
 
-	expect(result).toHaveRun("prismic", ["slice", "create"]);
+	expect(result).toHaveRun(["slice", "create"]);
 	const slices = await readLocalSlices(project);
 	expect(slices.length).toBe(1);
 	await expect(JSON.stringify(slices[0], null, 2)).toSatisfyJudge(
@@ -59,7 +59,7 @@ it.for(trials)(
 			`model a Prismic blog post: a title, publish date, hero image, author, and body`,
 		);
 
-		expect(result).toHaveRun("prismic", ["type", "create"]);
+		expect(result).toHaveRun(["type", "create"]);
 		const models = (await readLocalCustomTypes(project)).filter((model) => model.id !== author.id);
 
 		await expect(JSON.stringify(models, null, 2)).toSatisfyJudge(
@@ -76,7 +76,7 @@ it.for(trials)(
 	async (_, { project, agent, expect }) => {
 		const result = await agent(`Model a landing page and a global navigation menu.`);
 
-		expect(result).toHaveRun("prismic", ["type", "create"]);
+		expect(result).toHaveRun(["type", "create"]);
 		const models = await readLocalCustomTypes(project);
 		const landingPage = models.find((model) => /landing/.test(model.id));
 		const navigation = models.find((model) => /nav/.test(model.id));
@@ -95,7 +95,7 @@ it.for(trials)(
 
 		const result = await agent(`The homepage needs a flexible hero.`);
 
-		expect(result).toHaveRun("prismic");
+		expect(result).toHaveRun();
 		const models = await readLocalCustomTypes(project);
 		const slices = await readLocalSlices(project);
 		await expect(JSON.stringify({ models, slices }, null, 2)).toSatisfyJudge(
@@ -118,7 +118,7 @@ it.for(trials)(
 			`Add a star rating (1 to 5) and a call-to-action button to the "product" type.`,
 		);
 
-		expect(result).toHaveRun("prismic", ["field", "add"]);
+		expect(result).toHaveRun(["field", "add"]);
 		const model = await readLocalCustomType(project, product.id);
 		await expect(JSON.stringify(model, null, 2)).toSatisfyJudge(
 			dedent`
@@ -137,7 +137,7 @@ it.for(trials)(
 		const result = await agent(
 			`Set up the "blog_post" type: it needs a title and the author's Bluesky handle.`,
 		);
-		expect(result).toHaveRun("prismic", ["field", "add"]);
+		expect(result).toHaveRun(["field", "add"]);
 		const model = await readLocalCustomType(project, customType.id);
 
 		await expect(JSON.stringify(model, null, 2)).toSatisfyJudge(

evals/edit-models-precisely.eval.ts

@@ -23,7 +23,7 @@ it.for(trials)("adds a field", async (_, { project, agent, expect }) => {
 
 	const result = await agent(`Add an "excerpt" rich text field to the "article" type.`);
 
-	expect(result).toHaveRun("prismic", ["field", "add", "rich-text", "excerpt"]);
+	expect(result).toHaveRun(["field", "add", "rich-text", "excerpt"]);
 	const model = await readLocalCustomType(project, article.id);
 	expect(model.json.Main.excerpt.type).toBe("StructuredText");
 	expect(model.json.Main.title).toEqual(article.json.Main.title);
@@ -42,7 +42,7 @@ it.for(trials)(
 
 		const result = await agent(`Add a "centered" variation to the "Hero" slice.`);
 
-		expect(result).toHaveRun("prismic", ["slice", "add-variation"]);
+		expect(result).toHaveRun(["slice", "add-variation"]);
 		const model = await readLocalSlice(project, slice.id);
 		const ids = model?.variations.map((variation) => variation.id);
 		expect(ids).toContain("default");
@@ -68,7 +68,7 @@ it.todo("renames a field without disturbing field order", async ({ project, agen
 
 	const result = await agent(`Rename the "tagline" field on "article" to "subtitle".`);
 
-	expect(result).toHaveRun("prismic", ["field", "edit"]);
+	expect(result).toHaveRun(["field", "edit"]);
 	const model = await readLocalCustomType(project, article.id);
 	expect(Object.keys(model.json.Main)).toEqual(["title", "subtitle", "body"]);
 	expect(model.json.Main.subtitle.type).toBe("Text");
@@ -88,7 +88,7 @@ it.for(trials)(
 			`The testimonial design now also shows the author's company logo and a star rating. Update the "Testimonial" slice.`,
 		);
 
-		expect(result).toHaveRun("prismic", ["field", "add"]);
+		expect(result).toHaveRun(["field", "add"]);
 		const model = await readLocalSlice(project, slice.id);
 		const primary = model?.variations[0].primary ?? {};
 		expect(primary.quote).toEqual(slice.variations[0].primary.quote);
@@ -107,7 +107,7 @@ it.for(trials)(
 		const firstRun = await agent(`add a "body" rich text field to "homepage"`);
 		await agent(`add a "body" rich text field to "homepage"`);
 
-		expect(firstRun).toHaveRun("prismic", ["field", "add"]);
+		expect(firstRun).toHaveRun(["field", "add"]);
 		const model = await readLocalCustomType(project, homepage.id);
 		const bodyLikeKeys = Object.keys(model.json.Main).filter((key) => /body/i.test(key));
 		expect(bodyLikeKeys).toEqual(["body"]);
@@ -123,7 +123,7 @@ it.for(trials)("adds a group field with nested fields", async (_, { project, age
 		`Add a repeatable "features" group to the "product" type. Each feature has an icon image and a label.`,
 	);
 
-	expect(result).toHaveRun("prismic", ["field", "add", "group"]);
+	expect(result).toHaveRun(["field", "add", "group"]);
 	const model = await readLocalCustomType(project, product.id);
 	const features = model.json.Main.features;
 	expect(features.type).toBe("Group");
@@ -148,7 +148,7 @@ it.for(trials)(
 
 		// Both `field add content-relationship` and `field add link --allow document`
 		// model a constrained relationship; judge the resulting model, not the command.
-		expect(result).toHaveRun("prismic", ["field", "add"]);
+		expect(result).toHaveRun(["field", "add"]);
 		const model = await readLocalCustomType(project, product.id);
 		expect(model.json.Main.size.type).toBe("Select");
 		expect((model.json.Main.size.config as { options: string[] }).options).toEqual(["S", "M", "L"]);
@@ -180,7 +180,7 @@ it.for(trials)(
 			`Restrict the "title" field on "post" so editors can only write a single H1.`,
 		);
 
-		expect(result).toHaveRun("prismic", ["field", "edit"]);
+		expect(result).toHaveRun(["field", "edit"]);
 		const model = await readLocalCustomType(project, post.id);
 		const config = model.json.Main.title.config as { single?: string; multi?: string };
 		expect(config.single).toBe("heading1");
@@ -200,7 +200,7 @@ it.for(trials)("connects a slice to a page type", async (_, { project, agent, ex
 
 	const result = await agent(`Make the "Testimonial" slice available on the "page" type.`);
 
-	expect(result).toHaveRun("prismic", ["slice", "connect"]);
+	expect(result).toHaveRun(["slice", "connect"]);
 	const model = await readLocalCustomType(project, page.id);
 	const choices = (model.json.Main.slices.config as { choices: Record<string, unknown> }).choices;
 	expect(Object.keys(choices)).toContain(slice.id);

evals/initialize-projects.eval.ts

@@ -9,7 +9,7 @@ it.for(trials)(
 
 		const result = await agent(`Set up Prismic in this Next.js project.`);
 
-		expect(result).toHaveRun("prismic", ["init"]);
+		expect(result).toHaveRun(["init"]);
 		const config = JSON.parse(await readFile(new URL("prismic.config.json", project), "utf8"));
 		expect(config.repositoryName).toBeTruthy();
 	},
@@ -24,7 +24,7 @@ it.for(trials)(
 
 		const result = await agent(`Add Prismic to this existing Next.js app.`);
 
-		expect(result).toHaveRun("prismic", ["init"]);
+		expect(result).toHaveRun(["init"]);
 		const config = JSON.parse(await readFile(new URL("prismic.config.json", project), "utf8"));
 		expect(config.repositoryName).toBeTruthy();
 		const page = await readFile(new URL("app/page.tsx", project), "utf8");
@@ -41,7 +41,7 @@ it.for(trials)(
 			`Set up Prismic in this project using the existing "${repo}" Prismic repository.`,
 		);
 
-		expect(result).toHaveRun("prismic", ["init"]);
+		expect(result).toHaveRun(["init"]);
 		const config = JSON.parse(await readFile(new URL("prismic.config.json", project), "utf8"));
 		expect(config.repositoryName).toBe(repo);
 	},

evals/it.ts

@@ -1,7 +1,7 @@
 import { query, type SDKResultMessage } from "@anthropic-ai/claude-agent-sdk";
 import { Codex } from "@openai/codex-sdk";
 import dedent from "dedent";
-import { copyFile, mkdir, mkdtemp, readFile, symlink, writeFile } from "node:fs/promises";
+import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -35,7 +35,7 @@ declare module "vitest" {
 	}
 	// oxlint-disable-next-line no-explicit-any
 	interface Matchers<T = any> {
-		toHaveRun(bin: string, positionals?: string[]): T;
+		toHaveRun(positionals?: string[]): T;
 		toSatisfyJudge(criterion: string): Promise<T>;
 	}
 }
@@ -65,10 +65,21 @@ export const it = base.extend<{
 			CODEX_HOME: await createCodexHome(),
 		};
 
+		// The project's `prismic` bin records the argv of every call before running the
+		// CLI, so evals see the arguments as the CLI received them, not as the agent typed them.
+		const argvLog = join(tmpdir(), `prismic-argv-${crypto.randomUUID()}.jsonl`);
 		if (installCli) {
-			const nodeModulesBinDir = new URL("node_modules/.bin/", project);
-			await mkdir(nodeModulesBinDir, { recursive: true });
-			await symlink(BIN, new URL("prismic", nodeModulesBinDir));
+			const bin = new URL("node_modules/.bin/prismic", project);
+			await mkdir(new URL(".", bin), { recursive: true });
+			await writeFile(
+				bin,
+				dedent`
+					#!/bin/sh
+					node -e 'console.log(JSON.stringify(process.argv.slice(1)))' -- "$@" >> ${JSON.stringify(argvLog)}
+					exec node ${JSON.stringify(fileURLToPath(BIN))} "$@"
+				`,
+				{ mode: 0o755 },
+			);
 			env.PRISMIC_CONFIG_DIR = fileURLToPath(new URL(".config/prismic/", home));
 			env.PRISMIC_TYPE_BUILDER_ENABLED = "true";
 			env.PRISMIC_SENTRY_ENABLED = "false";
@@ -89,23 +100,22 @@ export const it = base.extend<{
 				skill: installSkill ? SKILL : undefined,
 				cwd: project,
 				env,
-				// Recorded as commands stream so a timed-out trial keeps its trail.
-				onCommand: (command) => {
-					commands.push(command);
-					if (/(^|\s)(npx\s+)?prismic(@|\s|$)/.test(command)) {
-						trial.calls.push(command.replace(/^.*?(^|\s)(npx\s+)?prismic(@\S+)?(?=\s|$)\s*/, ""));
-					}
-				},
+				onCommand: (command) => commands.push(command),
+			}).finally(async () => {
+				// Recorded even when the run fails so a timed-out trial keeps its trail.
+				durationMs += performance.now() - start;
+				trial.durationS = Math.round(durationMs / 1000);
+				trial.calls = await readArgvLog(argvLog);
 			});
 
-			durationMs += performance.now() - start;
 			trial.text = text;
 			trial.tokens += tokens;
-			trial.durationS = Math.round(durationMs / 1000);
 
-			return { text, commands };
+			return { text, commands, calls: trial.calls };
 		});
 
+		await rm(argvLog, { force: true });
+
 		for (const file of ["prismic.config.json", "slicemachine.config.json"]) {
 			try {
 				const configFile = await readFile(new URL(file, project), "utf8");
@@ -121,25 +131,21 @@ export const it = base.extend<{
 it.scoped({ isolateRepo: true });
 
 expect.extend({
-	toHaveRun(result: AgentResult, bin: string, positionals: string[] = []) {
-		const pass = result.commands.some((command) => {
-			return command.split(/&&|\|\||;|\||\n/).some((segment) => {
-				const words = segment.split(/\s+/).filter(Boolean);
-				if (words.includes("--help") || words.includes("-h")) return false;
-				const start = words.findIndex((word) => new RegExp(`^${bin}@?`).test(word));
-				if (start === -1) return false;
-
-				const got = words.slice(start + 1).filter((w) => !w.startsWith("-"));
-				return positionals.every((p, i) => got[i] === p);
-			});
+	toHaveRun(result: AgentResult, positionals: string[] = []) {
+		const pass = result.calls.some((argv) => {
+			if (argv.includes("--help") || argv.includes("-h")) return false;
+			const got = argv.filter((arg) => !arg.startsWith("-"));
+			return positionals.every((p, i) => got[i] === p);
 		});
 
 		return {
 			pass,
 			message: () => {
-				const wanted = [bin, ...positionals].join(" ");
+				const wanted = ["prismic", ...positionals].join(" ");
 				if (pass) return `expected no command matching \`${wanted}\`, but one ran`;
-				const seen = result.commands.map((c) => `  ${c}`).join("\n") || "  (no commands ran)";
+				const seen =
+					result.calls.map((argv) => `  prismic ${argv.join(" ")}`).join("\n") ||
+					"  (no commands ran)";
 				return `expected a command matching \`${wanted}\`, but saw:\n${seen}\n\nagent's final message:\n${result.text}`;
 			},
 		};
@@ -156,7 +162,10 @@ expect.extend({
 
 type AgentResult = {
 	text: string;
+	/** Every shell command the agent ran, as typed. */
 	commands: string[];
+	/** Every prismic CLI call, as the argv the CLI received. */
+	calls: string[][];
 };
 
 type RunOptions = {
@@ -293,6 +302,15 @@ async function judge(
 	return result.structured_output as { reason: string; pass: boolean };
 }
 
+async function readArgvLog(path: string): Promise<string[][]> {
+	const log = await readFile(path, "utf8").catch(() => "");
+	return log
+		.trim()
+		.split("\n")
+		.filter(Boolean)
+		.map((line) => JSON.parse(line));
+}
+
 async function fetchSkill() {
 	const response = await fetch(
 		`https://raw.githubusercontent.com/prismicio/skills/${PRISMIC_SKILL_REF}/skills/prismic/SKILL.md`,

evals/render-content.eval.ts

@@ -40,9 +40,9 @@ it.for(trials)(
 			`Build a Prismic landing page for this project: a "landing_page" type with a hero slice (heading, description, image, CTA), wired up so the page renders its slices.`,
 		);
 
-		expect(result).toHaveRun("prismic", ["type", "create"]);
-		expect(result).toHaveRun("prismic", ["slice", "create"]);
-		expect(result).toHaveRun("prismic", ["slice", "connect"]);
+		expect(result).toHaveRun(["type", "create"]);
+		expect(result).toHaveRun(["slice", "create"]);
+		expect(result).toHaveRun(["slice", "connect"]);
 		const models = await readLocalCustomTypes(project);
 		const landingPage = models.find((model) => model.id === "landing_page");
 		expect(landingPage).toBeTruthy();