diff --git a/evals/check-before-acting.eval.ts b/evals/check-before-acting.eval.ts index e869f14..7ed262d 100644 --- a/evals/check-before-acting.eval.ts +++ b/evals/check-before-acting.eval.ts @@ -16,7 +16,7 @@ it.for(trials)( `Set up content previews for this project so editors can preview drafts.`, ); - expect(result).toHaveRun("prismic", ["docs"]); + expect(result).toHaveRun(["docs"]); }, ); @@ -32,7 +32,7 @@ it.for(trials)( `Change the "getting-started" document from the "article" type to "blog_post".`, ); - expect(result).not.toHaveRun("prismic", ["type", "remove"]); + expect(result).not.toHaveRun(["type", "remove"]); const models = await readLocalCustomTypes(project); expect(models.find((model) => model.id === article.id)).toEqual(article); expect(models.find((model) => model.id === post.id)).toEqual(post); @@ -56,7 +56,7 @@ it.for(trials)( const result = await agent(`Clean up the fields on "article".`); - expect(result).not.toHaveRun("prismic", ["field", "remove"]); + expect(result).not.toHaveRun(["field", "remove"]); const model = await readLocalCustomType(project, article.id); expect(model.json.Main.title).toEqual(article.json.Main.title); expect(model.json.Main.body).toEqual(article.json.Main.body); diff --git a/evals/configure-repositories.eval.ts b/evals/configure-repositories.eval.ts index 7feda9e..e14b631 100644 --- a/evals/configure-repositories.eval.ts +++ b/evals/configure-repositories.eval.ts @@ -14,7 +14,7 @@ it.for(trials)("sets up a content preview", async (_, { agent, expect, repo, tok `Set up a content preview for this repo pointing at https://example.com/api/preview.`, ); - expect(result).toHaveRun("prismic", ["preview", "add"]); + expect(result).toHaveRun(["preview", "add"]); const previews = await getPreviews({ repo, token, host }); expect(previews.some((preview) => preview.url.includes("example.com"))).toBe(true); }); @@ -29,7 +29,7 @@ it.for(trials)( `We just deployed the site to https://example.com. Set up content previews for production.`, ); - expect(result).toHaveRun("prismic", ["preview", "add"]); + expect(result).toHaveRun(["preview", "add"]); const previews = await getPreviews({ repo, token, host }); expect(previews.some((preview) => preview.url.includes("example.com"))).toBe(true); expect(previews.some((preview) => preview.url.includes("localhost:3000"))).toBe(true); @@ -45,8 +45,8 @@ it.for(trials)( `Create a content API token named "ci" for this repo and make the content API private.`, ); - expect(result).toHaveRun("prismic", ["token", "create"]); - expect(result).toHaveRun("prismic", ["repo", "set-api-access"]); + expect(result).toHaveRun(["token", "create"]); + expect(result).toHaveRun(["repo", "set-api-access"]); const apps = await getAccessTokens({ repo, token, host }); expect(apps.some((app) => app.name === "ci")).toBe(true); }, @@ -57,7 +57,7 @@ it.for(trials)("registers a webhook", async (_, { agent, expect, repo, token, ho `Register a webhook at https://example.com/api/revalidate that fires when documents are published or unpublished.`, ); - expect(result).toHaveRun("prismic", ["webhook", "create"]); + expect(result).toHaveRun(["webhook", "create"]); const webhooks = await getWebhooks({ repo, token, host }); expect(JSON.stringify(webhooks)).toContain("example.com/api/revalidate"); }); @@ -65,7 +65,7 @@ it.for(trials)("registers a webhook", async (_, { agent, expect, repo, token, ho it.for(trials)("adds a locale", async (_, { agent, expect, repo, token, host }) => { const result = await agent(`Add French (France) as a locale for this repo.`); - expect(result).toHaveRun("prismic", ["locale", "add"]); + expect(result).toHaveRun(["locale", "add"]); const locales = await getLocales({ repo, token, host }); expect(locales.some((locale) => locale.id === "fr-fr")).toBe(true); }); diff --git a/evals/design-sensible-models.eval.ts b/evals/design-sensible-models.eval.ts index 1e24c8c..ec05f68 100644 --- a/evals/design-sensible-models.eval.ts +++ b/evals/design-sensible-models.eval.ts @@ -17,7 +17,7 @@ it.for(trials)( `Create a testimonial slice: a quote, author name, author role, an avatar, and a company logo.`, ); - expect(result).toHaveRun("prismic", ["slice", "create"]); + expect(result).toHaveRun(["slice", "create"]); const slices = await readLocalSlices(project); expect(slices.length).toBe(1); await expect(JSON.stringify(slices[0], null, 2)).toSatisfyJudge( @@ -37,7 +37,7 @@ it.for(trials)("models a slice from a screenshot", async (_, { project, agent, e const result = await agent(`Model a Prismic slice for the section design in feature-slice.png.`); - expect(result).toHaveRun("prismic", ["slice", "create"]); + expect(result).toHaveRun(["slice", "create"]); const slices = await readLocalSlices(project); expect(slices.length).toBe(1); await expect(JSON.stringify(slices[0], null, 2)).toSatisfyJudge( @@ -59,7 +59,7 @@ it.for(trials)( `model a Prismic blog post: a title, publish date, hero image, author, and body`, ); - expect(result).toHaveRun("prismic", ["type", "create"]); + expect(result).toHaveRun(["type", "create"]); const models = (await readLocalCustomTypes(project)).filter((model) => model.id !== author.id); await expect(JSON.stringify(models, null, 2)).toSatisfyJudge( @@ -76,7 +76,7 @@ it.for(trials)( async (_, { project, agent, expect }) => { const result = await agent(`Model a landing page and a global navigation menu.`); - expect(result).toHaveRun("prismic", ["type", "create"]); + expect(result).toHaveRun(["type", "create"]); const models = await readLocalCustomTypes(project); const landingPage = models.find((model) => /landing/.test(model.id)); const navigation = models.find((model) => /nav/.test(model.id)); @@ -95,7 +95,7 @@ it.for(trials)( const result = await agent(`The homepage needs a flexible hero.`); - expect(result).toHaveRun("prismic"); + expect(result).toHaveRun(); const models = await readLocalCustomTypes(project); const slices = await readLocalSlices(project); await expect(JSON.stringify({ models, slices }, null, 2)).toSatisfyJudge( @@ -118,7 +118,7 @@ it.for(trials)( `Add a star rating (1 to 5) and a call-to-action button to the "product" type.`, ); - expect(result).toHaveRun("prismic", ["field", "add"]); + expect(result).toHaveRun(["field", "add"]); const model = await readLocalCustomType(project, product.id); await expect(JSON.stringify(model, null, 2)).toSatisfyJudge( dedent` @@ -137,7 +137,7 @@ it.for(trials)( const result = await agent( `Set up the "blog_post" type: it needs a title and the author's Bluesky handle.`, ); - expect(result).toHaveRun("prismic", ["field", "add"]); + expect(result).toHaveRun(["field", "add"]); const model = await readLocalCustomType(project, customType.id); await expect(JSON.stringify(model, null, 2)).toSatisfyJudge( diff --git a/evals/edit-models-precisely.eval.ts b/evals/edit-models-precisely.eval.ts index 3680b9b..7a7fc9a 100644 --- a/evals/edit-models-precisely.eval.ts +++ b/evals/edit-models-precisely.eval.ts @@ -23,7 +23,7 @@ it.for(trials)("adds a field", async (_, { project, agent, expect }) => { const result = await agent(`Add an "excerpt" rich text field to the "article" type.`); - expect(result).toHaveRun("prismic", ["field", "add", "rich-text", "excerpt"]); + expect(result).toHaveRun(["field", "add", "rich-text", "excerpt"]); const model = await readLocalCustomType(project, article.id); expect(model.json.Main.excerpt.type).toBe("StructuredText"); expect(model.json.Main.title).toEqual(article.json.Main.title); @@ -42,7 +42,7 @@ it.for(trials)( const result = await agent(`Add a "centered" variation to the "Hero" slice.`); - expect(result).toHaveRun("prismic", ["slice", "add-variation"]); + expect(result).toHaveRun(["slice", "add-variation"]); const model = await readLocalSlice(project, slice.id); const ids = model?.variations.map((variation) => variation.id); expect(ids).toContain("default"); @@ -68,7 +68,7 @@ it.todo("renames a field without disturbing field order", async ({ project, agen const result = await agent(`Rename the "tagline" field on "article" to "subtitle".`); - expect(result).toHaveRun("prismic", ["field", "edit"]); + expect(result).toHaveRun(["field", "edit"]); const model = await readLocalCustomType(project, article.id); expect(Object.keys(model.json.Main)).toEqual(["title", "subtitle", "body"]); expect(model.json.Main.subtitle.type).toBe("Text"); @@ -88,7 +88,7 @@ it.for(trials)( `The testimonial design now also shows the author's company logo and a star rating. Update the "Testimonial" slice.`, ); - expect(result).toHaveRun("prismic", ["field", "add"]); + expect(result).toHaveRun(["field", "add"]); const model = await readLocalSlice(project, slice.id); const primary = model?.variations[0].primary ?? {}; expect(primary.quote).toEqual(slice.variations[0].primary.quote); @@ -107,7 +107,7 @@ it.for(trials)( const firstRun = await agent(`add a "body" rich text field to "homepage"`); await agent(`add a "body" rich text field to "homepage"`); - expect(firstRun).toHaveRun("prismic", ["field", "add"]); + expect(firstRun).toHaveRun(["field", "add"]); const model = await readLocalCustomType(project, homepage.id); const bodyLikeKeys = Object.keys(model.json.Main).filter((key) => /body/i.test(key)); expect(bodyLikeKeys).toEqual(["body"]); @@ -123,7 +123,7 @@ it.for(trials)("adds a group field with nested fields", async (_, { project, age `Add a repeatable "features" group to the "product" type. Each feature has an icon image and a label.`, ); - expect(result).toHaveRun("prismic", ["field", "add", "group"]); + expect(result).toHaveRun(["field", "add", "group"]); const model = await readLocalCustomType(project, product.id); const features = model.json.Main.features; expect(features.type).toBe("Group"); @@ -148,7 +148,7 @@ it.for(trials)( // Both `field add content-relationship` and `field add link --allow document` // model a constrained relationship; judge the resulting model, not the command. - expect(result).toHaveRun("prismic", ["field", "add"]); + expect(result).toHaveRun(["field", "add"]); const model = await readLocalCustomType(project, product.id); expect(model.json.Main.size.type).toBe("Select"); expect((model.json.Main.size.config as { options: string[] }).options).toEqual(["S", "M", "L"]); @@ -180,7 +180,7 @@ it.for(trials)( `Restrict the "title" field on "post" so editors can only write a single H1.`, ); - expect(result).toHaveRun("prismic", ["field", "edit"]); + expect(result).toHaveRun(["field", "edit"]); const model = await readLocalCustomType(project, post.id); const config = model.json.Main.title.config as { single?: string; multi?: string }; expect(config.single).toBe("heading1"); @@ -200,7 +200,7 @@ it.for(trials)("connects a slice to a page type", async (_, { project, agent, ex const result = await agent(`Make the "Testimonial" slice available on the "page" type.`); - expect(result).toHaveRun("prismic", ["slice", "connect"]); + expect(result).toHaveRun(["slice", "connect"]); const model = await readLocalCustomType(project, page.id); const choices = (model.json.Main.slices.config as { choices: Record }).choices; expect(Object.keys(choices)).toContain(slice.id); diff --git a/evals/initialize-projects.eval.ts b/evals/initialize-projects.eval.ts index e86f448..4d7400a 100644 --- a/evals/initialize-projects.eval.ts +++ b/evals/initialize-projects.eval.ts @@ -9,7 +9,7 @@ it.for(trials)( const result = await agent(`Set up Prismic in this Next.js project.`); - expect(result).toHaveRun("prismic", ["init"]); + expect(result).toHaveRun(["init"]); const config = JSON.parse(await readFile(new URL("prismic.config.json", project), "utf8")); expect(config.repositoryName).toBeTruthy(); }, @@ -24,7 +24,7 @@ it.for(trials)( const result = await agent(`Add Prismic to this existing Next.js app.`); - expect(result).toHaveRun("prismic", ["init"]); + expect(result).toHaveRun(["init"]); const config = JSON.parse(await readFile(new URL("prismic.config.json", project), "utf8")); expect(config.repositoryName).toBeTruthy(); const page = await readFile(new URL("app/page.tsx", project), "utf8"); @@ -41,7 +41,7 @@ it.for(trials)( `Set up Prismic in this project using the existing "${repo}" Prismic repository.`, ); - expect(result).toHaveRun("prismic", ["init"]); + expect(result).toHaveRun(["init"]); const config = JSON.parse(await readFile(new URL("prismic.config.json", project), "utf8")); expect(config.repositoryName).toBe(repo); }, diff --git a/evals/it.ts b/evals/it.ts index 97257f9..399702b 100644 --- a/evals/it.ts +++ b/evals/it.ts @@ -1,7 +1,7 @@ import { query, type SDKResultMessage } from "@anthropic-ai/claude-agent-sdk"; import { Codex } from "@openai/codex-sdk"; import dedent from "dedent"; -import { copyFile, mkdir, mkdtemp, readFile, symlink, writeFile } from "node:fs/promises"; +import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { homedir, tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -35,7 +35,7 @@ declare module "vitest" { } // oxlint-disable-next-line no-explicit-any interface Matchers { - toHaveRun(bin: string, positionals?: string[]): T; + toHaveRun(positionals?: string[]): T; toSatisfyJudge(criterion: string): Promise; } } @@ -65,10 +65,21 @@ export const it = base.extend<{ CODEX_HOME: await createCodexHome(), }; + // The project's `prismic` bin records the argv of every call before running the + // CLI, so evals see the arguments as the CLI received them, not as the agent typed them. + const argvLog = join(tmpdir(), `prismic-argv-${crypto.randomUUID()}.jsonl`); if (installCli) { - const nodeModulesBinDir = new URL("node_modules/.bin/", project); - await mkdir(nodeModulesBinDir, { recursive: true }); - await symlink(BIN, new URL("prismic", nodeModulesBinDir)); + const bin = new URL("node_modules/.bin/prismic", project); + await mkdir(new URL(".", bin), { recursive: true }); + await writeFile( + bin, + dedent` + #!/bin/sh + node -e 'console.log(JSON.stringify(process.argv.slice(1)))' -- "$@" >> ${JSON.stringify(argvLog)} + exec node ${JSON.stringify(fileURLToPath(BIN))} "$@" + `, + { mode: 0o755 }, + ); env.PRISMIC_CONFIG_DIR = fileURLToPath(new URL(".config/prismic/", home)); env.PRISMIC_TYPE_BUILDER_ENABLED = "true"; env.PRISMIC_SENTRY_ENABLED = "false"; @@ -89,23 +100,22 @@ export const it = base.extend<{ skill: installSkill ? SKILL : undefined, cwd: project, env, - // Recorded as commands stream so a timed-out trial keeps its trail. - onCommand: (command) => { - commands.push(command); - if (/(^|\s)(npx\s+)?prismic(@|\s|$)/.test(command)) { - trial.calls.push(command.replace(/^.*?(^|\s)(npx\s+)?prismic(@\S+)?(?=\s|$)\s*/, "")); - } - }, + onCommand: (command) => commands.push(command), + }).finally(async () => { + // Recorded even when the run fails so a timed-out trial keeps its trail. + durationMs += performance.now() - start; + trial.durationS = Math.round(durationMs / 1000); + trial.calls = await readArgvLog(argvLog); }); - durationMs += performance.now() - start; trial.text = text; trial.tokens += tokens; - trial.durationS = Math.round(durationMs / 1000); - return { text, commands }; + return { text, commands, calls: trial.calls }; }); + await rm(argvLog, { force: true }); + for (const file of ["prismic.config.json", "slicemachine.config.json"]) { try { const configFile = await readFile(new URL(file, project), "utf8"); @@ -121,25 +131,21 @@ export const it = base.extend<{ it.scoped({ isolateRepo: true }); expect.extend({ - toHaveRun(result: AgentResult, bin: string, positionals: string[] = []) { - const pass = result.commands.some((command) => { - return command.split(/&&|\|\||;|\||\n/).some((segment) => { - const words = segment.split(/\s+/).filter(Boolean); - if (words.includes("--help") || words.includes("-h")) return false; - const start = words.findIndex((word) => new RegExp(`^${bin}@?`).test(word)); - if (start === -1) return false; - - const got = words.slice(start + 1).filter((w) => !w.startsWith("-")); - return positionals.every((p, i) => got[i] === p); - }); + toHaveRun(result: AgentResult, positionals: string[] = []) { + const pass = result.calls.some((argv) => { + if (argv.includes("--help") || argv.includes("-h")) return false; + const got = argv.filter((arg) => !arg.startsWith("-")); + return positionals.every((p, i) => got[i] === p); }); return { pass, message: () => { - const wanted = [bin, ...positionals].join(" "); + const wanted = ["prismic", ...positionals].join(" "); if (pass) return `expected no command matching \`${wanted}\`, but one ran`; - const seen = result.commands.map((c) => ` ${c}`).join("\n") || " (no commands ran)"; + const seen = + result.calls.map((argv) => ` prismic ${argv.join(" ")}`).join("\n") || + " (no commands ran)"; return `expected a command matching \`${wanted}\`, but saw:\n${seen}\n\nagent's final message:\n${result.text}`; }, }; @@ -156,7 +162,10 @@ expect.extend({ type AgentResult = { text: string; + /** Every shell command the agent ran, as typed. */ commands: string[]; + /** Every prismic CLI call, as the argv the CLI received. */ + calls: string[][]; }; type RunOptions = { @@ -293,6 +302,15 @@ async function judge( return result.structured_output as { reason: string; pass: boolean }; } +async function readArgvLog(path: string): Promise { + const log = await readFile(path, "utf8").catch(() => ""); + return log + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); +} + async function fetchSkill() { const response = await fetch( `https://raw.githubusercontent.com/prismicio/skills/${PRISMIC_SKILL_REF}/skills/prismic/SKILL.md`, diff --git a/evals/render-content.eval.ts b/evals/render-content.eval.ts index d27aa1e..6c9ed5f 100644 --- a/evals/render-content.eval.ts +++ b/evals/render-content.eval.ts @@ -40,9 +40,9 @@ it.for(trials)( `Build a Prismic landing page for this project: a "landing_page" type with a hero slice (heading, description, image, CTA), wired up so the page renders its slices.`, ); - expect(result).toHaveRun("prismic", ["type", "create"]); - expect(result).toHaveRun("prismic", ["slice", "create"]); - expect(result).toHaveRun("prismic", ["slice", "connect"]); + expect(result).toHaveRun(["type", "create"]); + expect(result).toHaveRun(["slice", "create"]); + expect(result).toHaveRun(["slice", "connect"]); const models = await readLocalCustomTypes(project); const landingPage = models.find((model) => model.id === "landing_page"); expect(landingPage).toBeTruthy(); diff --git a/evals/repair-generated-files.eval.ts b/evals/repair-generated-files.eval.ts index b0ef2b0..a1b932b 100644 --- a/evals/repair-generated-files.eval.ts +++ b/evals/repair-generated-files.eval.ts @@ -17,7 +17,7 @@ it.for(trials)( `The components map in slices/index.js is out of date and my slice does not render. Fix it.`, ); - expect(result).toHaveRun("prismic", ["gen", "slice-index"]); + expect(result).toHaveRun(["gen", "slice-index"]); expect(await readFile(new URL("slices/index.js", project), "utf8")).toContain(slice.id); }, ); diff --git a/evals/reporter.ts b/evals/reporter.ts index b651611..75ec1dc 100644 --- a/evals/reporter.ts +++ b/evals/reporter.ts @@ -11,8 +11,8 @@ export type Trial = { tokens: number; /** Agent wall time in seconds, excluding fixture setup and judging. */ durationS: number; - /** prismic CLI invocations, verbatim minus the leading `npx prismic`. */ - calls: string[]; + /** prismic CLI invocations, each as the argv the CLI received. */ + calls: string[][]; /** The agent's final message, the only record of why it stopped. */ text: string; }; diff --git a/evals/sync-models.eval.ts b/evals/sync-models.eval.ts index 1c8dc0f..f9c1013 100644 --- a/evals/sync-models.eval.ts +++ b/evals/sync-models.eval.ts @@ -22,8 +22,8 @@ it.for(trials)( `The models in this Prismic repo were updated by a teammate. Bring this project up to date.`, ); - expect(result).toHaveRun("prismic", ["pull"]); - expect(result).not.toHaveRun("prismic", ["push"]); + expect(result).toHaveRun(["pull"]); + expect(result).not.toHaveRun(["push"]); const local = await readLocalCustomType(project, article.id); expect(local.json.Main.subtitle).toEqual(subtitle); const remoteTypes = await getCustomTypes({ repo, token, host }); @@ -55,7 +55,7 @@ it.for(trials)( `I finished modeling the "article" type. Publish it so editors can start using it.`, ); - expect(result).toHaveRun("prismic", ["push"]); + expect(result).toHaveRun(["push"]); const remoteTypes = await getCustomTypes({ repo, token, host }); expect(remoteTypes.some((type) => type.id === article.id)).toBe(true); const status = await exec("git", ["status", "--porcelain", "customtypes"]);