diff --git a/src/judges/README.md b/src/judges/README.md index 0bba170..1b02f07 100644 --- a/src/judges/README.md +++ b/src/judges/README.md @@ -43,11 +43,15 @@ interface ModelConfig { displayName: string supportsTemperature: boolean defaultTemperature: number - maxTokensParam: "maxTokens" | "max_completion_tokens" defaultMaxTokens: number } ``` +`defaultMaxTokens` is passed to the AI SDK as `maxOutputTokens`, which the SDK maps to each +provider's own parameter — there is no per-provider parameter name to configure. Give +reasoning/thinking models a roomy value: their reasoning tokens are billed against the same +ceiling, so a tight cap can be spent before any visible answer is produced. + ## Provider-Specific Prompts Providers can override judge prompts. See [providers/README.md](../providers/README.md#custom-prompts). diff --git a/src/judges/anthropic.ts b/src/judges/anthropic.ts index e8cad9e..02e22f1 100644 --- a/src/judges/anthropic.ts +++ b/src/judges/anthropic.ts @@ -27,17 +27,14 @@ export class AnthropicJudge implements Judge { const prompt = buildJudgePrompt(input) - const params: Record = { + const { text } = await generateText({ model: this.client(this.modelConfig.id), prompt, - maxTokens: this.modelConfig.defaultMaxTokens, - } - - if (this.modelConfig.supportsTemperature) { - params.temperature = this.modelConfig.defaultTemperature - } - - const { text } = await generateText(params as Parameters[0]) + maxOutputTokens: this.modelConfig.defaultMaxTokens, + ...(this.modelConfig.supportsTemperature + ? { temperature: this.modelConfig.defaultTemperature } + : {}), + }) return parseJudgeResponse(text) } diff --git a/src/judges/base.ts b/src/judges/base.ts index bfdddae..692589f 100644 --- a/src/judges/base.ts +++ b/src/judges/base.ts @@ -28,6 +28,16 @@ System's Hypothesis: ${input.hypothesis}` } export function parseJudgeResponse(response: string): JudgeResult { + // An empty completion is not a verdict. Now that maxOutputTokens is actually enforced, + // a reasoning model can spend its whole ceiling on reasoning and return nothing; scoring + // that as "incorrect" would silently mark questions wrong and skew the run's accuracy. + // Throwing lets the evaluate phase record a real failure that a resume can retry. + if (!response.trim()) { + throw new Error( + "Judge returned an empty response (likely truncated before producing a verdict — check the model's maxOutputTokens)" + ) + } + try { const jsonMatch = response.match(/\{[\s\S]*\}/) if (!jsonMatch) { diff --git a/src/judges/google.ts b/src/judges/google.ts index 868dbb7..c304ed1 100644 --- a/src/judges/google.ts +++ b/src/judges/google.ts @@ -27,17 +27,14 @@ export class GoogleJudge implements Judge { const prompt = buildJudgePrompt(input) - const params: Record = { + const { text } = await generateText({ model: this.client(this.modelConfig.id), prompt, - maxTokens: this.modelConfig.defaultMaxTokens, - } - - if (this.modelConfig.supportsTemperature) { - params.temperature = this.modelConfig.defaultTemperature - } - - const { text } = await generateText(params as Parameters[0]) + maxOutputTokens: this.modelConfig.defaultMaxTokens, + ...(this.modelConfig.supportsTemperature + ? { temperature: this.modelConfig.defaultTemperature } + : {}), + }) return parseJudgeResponse(text) } diff --git a/src/judges/openai.ts b/src/judges/openai.ts index 0d6a0e0..c8b3080 100644 --- a/src/judges/openai.ts +++ b/src/judges/openai.ts @@ -27,18 +27,16 @@ export class OpenAIJudge implements Judge { const prompt = buildJudgePrompt(input) - const params: Record = { + // No `as Parameters[0]` cast here: it suppressed excess-property + // checking, which is why the v4 `maxTokens` name survived the AI SDK v5 upgrade unnoticed. + const { text } = await generateText({ model: this.client(this.modelConfig.id), prompt, - } - - if (this.modelConfig.supportsTemperature) { - params.temperature = this.modelConfig.defaultTemperature - } - - params.maxTokens = this.modelConfig.defaultMaxTokens - - const { text } = await generateText(params as Parameters[0]) + maxOutputTokens: this.modelConfig.defaultMaxTokens, + ...(this.modelConfig.supportsTemperature + ? { temperature: this.modelConfig.defaultTemperature } + : {}), + }) return parseJudgeResponse(text) } diff --git a/src/orchestrator/phases/answer.ts b/src/orchestrator/phases/answer.ts index d9e6df7..5c8b763 100644 --- a/src/orchestrator/phases/answer.ts +++ b/src/orchestrator/phases/answer.ts @@ -129,17 +129,14 @@ export async function runAnswerPhase( // custom prompt functions that transform context (e.g. Zep's XML-like tags). const contextTokens = Math.max(0, promptTokens - basePromptTokens) - const params: Record = { + const { text } = await generateText({ model: client(modelConfig.id), prompt, - maxTokens: modelConfig.defaultMaxTokens, - } - - if (modelConfig.supportsTemperature) { - params.temperature = modelConfig.defaultTemperature - } - - const { text } = await generateText(params as Parameters[0]) + maxOutputTokens: modelConfig.defaultMaxTokens, + ...(modelConfig.supportsTemperature + ? { temperature: modelConfig.defaultTemperature } + : {}), + }) const durationMs = Date.now() - startTime checkpointManager.updatePhase(checkpoint, question.questionId, "answer", { diff --git a/src/prompts/extraction.ts b/src/prompts/extraction.ts index daca2b9..ca21f66 100644 --- a/src/prompts/extraction.ts +++ b/src/prompts/extraction.ts @@ -1,9 +1,17 @@ import { createOpenAI } from "@ai-sdk/openai" import { generateText } from "ai" import type { UnifiedSession } from "../types/unified" +import { logger } from "../utils/logger" /** Model used for memory extraction (fast, cheap, sufficient for extraction) */ const EXTRACTION_MODEL = "gpt-4o-mini" +/** + * A long session can yield dozens of bullets, and this ceiling is now actually enforced + * (it previously used the AI SDK v4 `maxTokens` name and was silently dropped), so it needs + * real headroom: a truncated extraction quietly drops memories from the corpus the + * filesystem/rag providers are scored on. + */ +const EXTRACTION_MAX_TOKENS = 8000 /** * Build an extraction prompt that instructs the LLM to extract structured @@ -74,14 +82,20 @@ export async function extractMemories( ): Promise { const prompt = buildExtractionPrompt(session) - const params: Record = { + const { text, finishReason } = await generateText({ model: openai(EXTRACTION_MODEL), prompt, - maxTokens: 2000, + maxOutputTokens: EXTRACTION_MAX_TOKENS, temperature: 0, + }) + + // Truncation here silently shrinks the memory corpus, which reads as a provider + // quality problem rather than a harness limit. Say so instead. + if (finishReason === "length") { + logger.warn( + `Memory extraction for session ${session.sessionId} hit the ${EXTRACTION_MAX_TOKENS}-token ceiling and was truncated; some memories are missing.` + ) } - const { text } = await generateText(params as Parameters[0]) - return text.trim() } diff --git a/src/utils/models.test.ts b/src/utils/models.test.ts new file mode 100644 index 0000000..352b5da --- /dev/null +++ b/src/utils/models.test.ts @@ -0,0 +1,35 @@ +import { test, expect } from "bun:test" +import { MODEL_CONFIGS, getModelConfig } from "./models" + +// Reasoning/thinking models are billed for reasoning tokens against the same ceiling as +// visible output, so a tight maxOutputTokens can be consumed before any answer is emitted. +// An empty judge completion would mark questions incorrect, so these need real headroom. +const NEEDS_HEADROOM = /^(gpt-5|o1|o3|o4|gemini-2\.5|gemini-3)/ +const HEADROOM_FLOOR = 25000 + +test("every model config declares a positive output ceiling", () => { + for (const [alias, config] of Object.entries(MODEL_CONFIGS)) { + expect(config.defaultMaxTokens, alias).toBeGreaterThan(0) + } +}) + +test("reasoning and thinking models get enough headroom to emit a verdict", () => { + for (const [alias, config] of Object.entries(MODEL_CONFIGS)) { + if (NEEDS_HEADROOM.test(alias) || !config.supportsTemperature) { + expect(config.defaultMaxTokens, alias).toBeGreaterThanOrEqual(HEADROOM_FLOOR) + } + } +}) + +test("unknown models fall back to a ceiling that cannot truncate a verdict", () => { + // We cannot tell whether an unrecognised model reasons, so the fallback must be roomy. + for (const alias of ["gpt-5.5", "o5-mini", "gpt-4.7", "claude-opus-5", "gemini-4-pro"]) { + expect(getModelConfig(alias).defaultMaxTokens, alias).toBeGreaterThanOrEqual(HEADROOM_FLOOR) + } +}) + +test("non-reasoning models stay capped tightly enough to be worth capping", () => { + for (const alias of ["gpt-4o", "gpt-4.1-mini", "sonnet-4", "opus-4.5"]) { + expect(getModelConfig(alias).defaultMaxTokens, alias).toBeLessThanOrEqual(4000) + } +}) diff --git a/src/utils/models.ts b/src/utils/models.ts index b29ac80..8ced4b1 100644 --- a/src/utils/models.ts +++ b/src/utils/models.ts @@ -4,10 +4,25 @@ export interface ModelConfig { displayName: string supportsTemperature: boolean defaultTemperature: number - maxTokensParam: "maxTokens" | "max_completion_tokens" | "maxOutputTokens" + /** + * Ceiling passed as the AI SDK's `maxOutputTokens`. This is a runaway guard, not a + * budget: judge verdicts and benchmark answers are short, so a normal call uses a + * fraction of it. + * + * Reasoning/thinking models need far more headroom than the visible answer suggests. + * The SDK forwards this as `max_completion_tokens` for OpenAI reasoning models, which + * counts reasoning tokens *and* visible output, so a tight cap can be spent entirely + * on internal reasoning and return empty text. An empty judge response would score + * every question "incorrect", so these get ROOMY_MAX_TOKENS. + */ defaultMaxTokens: number } +/** Enough for a short answer or verdict on a non-reasoning model. */ +const SHORT_MAX_TOKENS = 1000 +/** Leaves room for reasoning/thinking tokens that are billed against the same ceiling. */ +const ROOMY_MAX_TOKENS = 25000 + export const MODEL_CONFIGS: Record = { // OpenAI - Standard models (support temperature) "gpt-4o": { @@ -16,8 +31,7 @@ export const MODEL_CONFIGS: Record = { displayName: "GPT-4o (Legacy)", supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: SHORT_MAX_TOKENS, }, "gpt-4o-mini": { id: "gpt-4o-mini", @@ -25,8 +39,7 @@ export const MODEL_CONFIGS: Record = { displayName: "GPT-4o Mini (Legacy)", supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: SHORT_MAX_TOKENS, }, "gpt-4.1": { id: "gpt-4.1", @@ -34,8 +47,7 @@ export const MODEL_CONFIGS: Record = { displayName: "GPT-4.1", supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: SHORT_MAX_TOKENS, }, "gpt-4.1-mini": { id: "gpt-4.1-mini", @@ -43,8 +55,7 @@ export const MODEL_CONFIGS: Record = { displayName: "GPT-4.1 Mini", supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: SHORT_MAX_TOKENS, }, "gpt-4.1-nano": { id: "gpt-4.1-nano", @@ -52,8 +63,7 @@ export const MODEL_CONFIGS: Record = { displayName: "GPT-4.1 Nano", supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: SHORT_MAX_TOKENS, }, // OpenAI - Reasoning models (NO temperature support) @@ -63,8 +73,7 @@ export const MODEL_CONFIGS: Record = { displayName: "GPT-5", supportsTemperature: false, defaultTemperature: 1, - maxTokensParam: "max_completion_tokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, }, "gpt-5-mini": { id: "gpt-5-mini", @@ -72,8 +81,7 @@ export const MODEL_CONFIGS: Record = { displayName: "GPT-5 Mini", supportsTemperature: false, defaultTemperature: 1, - maxTokensParam: "max_completion_tokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, }, o1: { id: "o1", @@ -81,8 +89,7 @@ export const MODEL_CONFIGS: Record = { displayName: "o1", supportsTemperature: false, defaultTemperature: 1, - maxTokensParam: "max_completion_tokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, }, "o1-pro": { id: "o1-pro", @@ -90,8 +97,7 @@ export const MODEL_CONFIGS: Record = { displayName: "o1 Pro", supportsTemperature: false, defaultTemperature: 1, - maxTokensParam: "max_completion_tokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, }, o3: { id: "o3", @@ -99,8 +105,7 @@ export const MODEL_CONFIGS: Record = { displayName: "o3", supportsTemperature: false, defaultTemperature: 1, - maxTokensParam: "max_completion_tokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, }, "o3-mini": { id: "o3-mini", @@ -108,8 +113,7 @@ export const MODEL_CONFIGS: Record = { displayName: "o3 Mini", supportsTemperature: false, defaultTemperature: 1, - maxTokensParam: "max_completion_tokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, }, "o3-pro": { id: "o3-pro", @@ -117,8 +121,7 @@ export const MODEL_CONFIGS: Record = { displayName: "o3 Pro", supportsTemperature: false, defaultTemperature: 1, - maxTokensParam: "max_completion_tokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, }, "o4-mini": { id: "o4-mini", @@ -126,8 +129,7 @@ export const MODEL_CONFIGS: Record = { displayName: "o4 Mini", supportsTemperature: false, defaultTemperature: 1, - maxTokensParam: "max_completion_tokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, }, // Anthropic - All Claude models (support temperature) @@ -137,8 +139,7 @@ export const MODEL_CONFIGS: Record = { displayName: "Claude Opus 4.5", supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: SHORT_MAX_TOKENS, }, "sonnet-4.5": { id: "claude-sonnet-4-5-20250929", @@ -146,8 +147,7 @@ export const MODEL_CONFIGS: Record = { displayName: "Claude Sonnet 4.5", supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: SHORT_MAX_TOKENS, }, "haiku-4.5": { id: "claude-haiku-4-5-20251001", @@ -155,8 +155,7 @@ export const MODEL_CONFIGS: Record = { displayName: "Claude Haiku 4.5", supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: SHORT_MAX_TOKENS, }, "opus-4.1": { id: "claude-opus-4-1-20250805", @@ -164,8 +163,7 @@ export const MODEL_CONFIGS: Record = { displayName: "Claude Opus 4.1", supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: SHORT_MAX_TOKENS, }, "sonnet-4": { id: "claude-sonnet-4-20250514", @@ -173,8 +171,7 @@ export const MODEL_CONFIGS: Record = { displayName: "Claude Sonnet 4", supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: SHORT_MAX_TOKENS, }, // Google - Gemini 2.x (support temperature) @@ -184,8 +181,7 @@ export const MODEL_CONFIGS: Record = { displayName: "Gemini 2.5 Pro", supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, }, "gemini-2.5-flash": { id: "gemini-2.5-flash", @@ -193,8 +189,7 @@ export const MODEL_CONFIGS: Record = { displayName: "Gemini 2.5 Flash", supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, }, "gemini-2.5-flash-lite": { id: "gemini-2.5-flash-lite", @@ -202,8 +197,7 @@ export const MODEL_CONFIGS: Record = { displayName: "Gemini 2.5 Flash Lite", supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, }, "gemini-2.0-flash": { id: "gemini-2.0-flash", @@ -211,8 +205,7 @@ export const MODEL_CONFIGS: Record = { displayName: "Gemini 2.0 Flash", supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: SHORT_MAX_TOKENS, }, // Google - Gemini 3 (MUST use temperature=1, lower causes issues) @@ -222,8 +215,7 @@ export const MODEL_CONFIGS: Record = { displayName: "Gemini 3 Pro Preview", supportsTemperature: true, defaultTemperature: 1, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, }, } @@ -241,7 +233,9 @@ export function getModelConfig(alias: string): ModelConfig { return MODEL_CONFIGS[lowerAlias] } - // Fallback for unknown models - try to infer from prefix + // Fallback for unknown models - try to infer from prefix. + // Unknown models get ROOMY_MAX_TOKENS: we cannot tell whether they reason, and a ceiling + // that truncates a judge silently corrupts scores, while a loose one only costs tokens. if ( alias.startsWith("gpt-5") || alias.startsWith("o1") || @@ -254,8 +248,7 @@ export function getModelConfig(alias: string): ModelConfig { displayName: alias, supportsTemperature: false, defaultTemperature: 1, - maxTokensParam: "max_completion_tokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, } } if (alias.startsWith("gpt-")) { @@ -265,8 +258,7 @@ export function getModelConfig(alias: string): ModelConfig { displayName: alias, supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, } } if (alias.startsWith("claude-")) { @@ -276,8 +268,7 @@ export function getModelConfig(alias: string): ModelConfig { displayName: alias, supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, } } if (alias.startsWith("gemini-3")) { @@ -287,8 +278,7 @@ export function getModelConfig(alias: string): ModelConfig { displayName: alias, supportsTemperature: true, defaultTemperature: 1, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, } } if (alias.startsWith("gemini-")) { @@ -298,20 +288,19 @@ export function getModelConfig(alias: string): ModelConfig { displayName: alias, supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, } } - // Default fallback + // Default fallback. Also roomy: a future reasoning model (say "o5-mini") matches none of + // the prefixes above and lands here, and starving it would silently void its verdicts. return { id: alias, provider: "openai", displayName: alias, supportsTemperature: true, defaultTemperature: 0, - maxTokensParam: "maxTokens", - defaultMaxTokens: 1000, + defaultMaxTokens: ROOMY_MAX_TOKENS, } }