diff --git a/src/orchestrator/batch.ts b/src/orchestrator/batch.ts index 239c6ae..c019b34 100644 --- a/src/orchestrator/batch.ts +++ b/src/orchestrator/batch.ts @@ -1,7 +1,7 @@ import type { ProviderName } from "../types/provider" import type { BenchmarkName } from "../types/benchmark" import type { SamplingConfig } from "../types/checkpoint" -import type { BenchmarkResult } from "../types/unified" +import type { BenchmarkResult, RetrievalAggregates } from "../types/unified" import { orchestrator, CheckpointManager } from "./index" import { createBenchmark } from "../benchmarks" import { logger } from "../utils/logger" @@ -432,116 +432,46 @@ export class BatchManager { const hasRetrieval = reports.some((r) => r.report.retrieval) if (hasRetrieval) { const k = reports.find((r) => r.report.retrieval)?.report.retrieval?.k || 10 - console.log(`\nRETRIEVAL METRICS (K=${k})`) - console.log( - "┌" + - "─".repeat(17) + - "┬" + - "─".repeat(9) + - "┬" + - "─".repeat(11) + - "┬" + - "─".repeat(10) + - "┬" + - "─".repeat(9) + - "┬" + - "─".repeat(9) + - "┬" + - "─".repeat(9) + - "┐" - ) - console.log( - "│ " + - pad("Provider", 15) + - " │ " + - pad("Hit@K", 7) + - " │ " + - pad("Precision", 9) + - " │ " + - pad("Recall", 8) + - " │ " + - pad("F1", 7) + - " │ " + - pad("MRR", 7) + - " │ " + - pad("NDCG", 7) + - " │" - ) - console.log( - "├" + - "─".repeat(17) + - "┼" + - "─".repeat(9) + - "┼" + - "─".repeat(11) + - "┼" + - "─".repeat(10) + - "┼" + - "─".repeat(9) + - "┼" + - "─".repeat(9) + - "┼" + - "─".repeat(9) + - "┤" - ) + console.log(` +RETRIEVAL METRICS (K=${k})`) + + // Recall/F1/NDCG used to sit in this table but were degenerate without ground-truth + // relevance counts (see #67), so the columns are derived from one list rather than + // repeated across four border strings and two row branches. + const columns: Array<{ + header: string + width: number + value: (r: RetrievalAggregates) => string + }> = [ + { header: "Provider", width: 15, value: () => "" }, + { header: "Hit@K", width: 7, value: (r) => padPct(r.hitAtK, 7) }, + { header: "Precision", width: 9, value: (r) => padPct(r.precisionAtK, 9) }, + { header: "MRR", width: 7, value: (r) => r.mrr.toFixed(3).padStart(7) }, + ] + + const border = (left: string, mid: string, right: string) => + left + columns.map((c) => "─".repeat(c.width + 2)).join(mid) + right + const row = (cells: string[]) => "│ " + cells.join(" │ ") + " │" + + console.log(border("┌", "┬", "┐")) + console.log(row(columns.map((c) => pad(c.header, c.width)))) + console.log(border("├", "┼", "┤")) for (const { provider, report } of reports) { - if (report.retrieval) { - const r = report.retrieval - console.log( - "│ " + - pad(provider, 15) + - " │ " + - padPct(r.hitAtK, 7) + - " │ " + - padPct(r.precisionAtK, 9) + - " │ " + - padPct(r.recallAtK, 8) + - " │ " + - padPct(r.f1AtK, 7) + - " │ " + - r.mrr.toFixed(3).padStart(7) + - " │ " + - r.ndcg.toFixed(3).padStart(7) + - " │" + const retrieval = report.retrieval + console.log( + row( + columns.map((c, i) => + i === 0 + ? pad(provider, c.width) + : retrieval + ? c.value(retrieval) + : pad("N/A", c.width) + ) ) - } else { - console.log( - "│ " + - pad(provider, 15) + - " │ " + - pad("N/A", 7) + - " │ " + - pad("N/A", 9) + - " │ " + - pad("N/A", 8) + - " │ " + - pad("N/A", 7) + - " │ " + - pad("N/A", 7) + - " │ " + - pad("N/A", 7) + - " │" - ) - } + ) } - console.log( - "└" + - "─".repeat(17) + - "┴" + - "─".repeat(9) + - "┴" + - "─".repeat(11) + - "┴" + - "─".repeat(10) + - "┴" + - "─".repeat(9) + - "┴" + - "─".repeat(9) + - "┴" + - "─".repeat(9) + - "┘" - ) + console.log(border("└", "┴", "┘")) } const allTypes = new Set() diff --git a/src/orchestrator/phases/report.ts b/src/orchestrator/phases/report.ts index 4ec9aab..0ed8ce8 100644 --- a/src/orchestrator/phases/report.ts +++ b/src/orchestrator/phases/report.ts @@ -22,23 +22,17 @@ function aggregateRetrievalMetrics(metrics: RetrievalMetrics[]): RetrievalAggreg (acc, m) => ({ hitAtK: acc.hitAtK + m.hitAtK, precisionAtK: acc.precisionAtK + m.precisionAtK, - recallAtK: acc.recallAtK + m.recallAtK, - f1AtK: acc.f1AtK + m.f1AtK, mrr: acc.mrr + m.mrr, - ndcg: acc.ndcg + m.ndcg, k: m.k, }), - { hitAtK: 0, precisionAtK: 0, recallAtK: 0, f1AtK: 0, mrr: 0, ndcg: 0, k: 10 } + { hitAtK: 0, precisionAtK: 0, mrr: 0, k: 10 } ) const n = metrics.length return { hitAtK: sum.hitAtK / n, precisionAtK: sum.precisionAtK / n, - recallAtK: sum.recallAtK / n, - f1AtK: sum.f1AtK / n, mrr: sum.mrr / n, - ndcg: sum.ndcg / n, k: sum.k, } } @@ -341,10 +335,7 @@ export function printReport(result: BenchmarkResult): void { console.log("\nRETRIEVAL QUALITY (K=" + result.retrieval.k + "):") console.log(` Hit@K: ${(result.retrieval.hitAtK * 100).toFixed(1)}%`) console.log(` Precision: ${(result.retrieval.precisionAtK * 100).toFixed(1)}%`) - console.log(` Recall: ${(result.retrieval.recallAtK * 100).toFixed(1)}%`) - console.log(` F1: ${(result.retrieval.f1AtK * 100).toFixed(1)}%`) console.log(` MRR: ${result.retrieval.mrr.toFixed(3)}`) - console.log(` NDCG: ${result.retrieval.ndcg.toFixed(3)}`) } console.log("-".repeat(60)) @@ -361,7 +352,7 @@ export function printReport(result: BenchmarkResult): void { ) if (stats.retrieval) { console.log( - ` Retrieval: Hit@${stats.retrieval.k}=${(stats.retrieval.hitAtK * 100).toFixed(0)}%, P=${(stats.retrieval.precisionAtK * 100).toFixed(0)}%, R=${(stats.retrieval.recallAtK * 100).toFixed(0)}%, MRR=${stats.retrieval.mrr.toFixed(2)}` + ` Retrieval: Hit@${stats.retrieval.k}=${(stats.retrieval.hitAtK * 100).toFixed(0)}%, P=${(stats.retrieval.precisionAtK * 100).toFixed(0)}%, MRR=${stats.retrieval.mrr.toFixed(2)}` ) } } diff --git a/src/orchestrator/phases/retrieval-eval.test.ts b/src/orchestrator/phases/retrieval-eval.test.ts new file mode 100644 index 0000000..398c5eb --- /dev/null +++ b/src/orchestrator/phases/retrieval-eval.test.ts @@ -0,0 +1,121 @@ +import { test, expect } from "bun:test" +import type { LanguageModel } from "ai" +import { calculateRetrievalMetrics } from "./retrieval-eval" + +// A minimal LanguageModelV2 stub. `ai/test`'s MockLanguageModelV2 would do this for us but it +// pulls in msw, which isn't a dependency of this project — and a hand-rolled stub is 10 lines. +function stubJudge(respond: () => string): LanguageModel { + return { + specificationVersion: "v2", + provider: "stub", + modelId: "stub-judge", + supportedUrls: {}, + doGenerate: async () => ({ + content: [{ type: "text" as const, text: respond() }], + finishReason: "stop" as const, + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + warnings: [], + }), + doStream: async () => { + throw new Error("not used") + }, + } as unknown as LanguageModel +} + +const judgeReturning = (text: string) => stubJudge(() => text) +const judgeThatFails = () => + stubJudge(() => { + throw new Error("429 rate limited") + }) + +const RESULTS = [{ chunk: "a" }, { chunk: "b" }, { chunk: "c" }, { chunk: "d" }] + +test("reports only metrics that are well defined without ground-truth relevance counts", async () => { + // 2nd and 4th results relevant. + const judge = judgeReturning( + JSON.stringify([ + { id: "result_1", relevant: 0 }, + { id: "result_2", relevant: 1 }, + { id: "result_3", relevant: 0 }, + { id: "result_4", relevant: 1 }, + ]) + ) + + const metrics = await calculateRetrievalMetrics(judge, "q", "gt", RESULTS) + + expect(metrics).toEqual({ + hitAtK: 1, + precisionAtK: 0.5, + mrr: 0.5, // first relevant at rank 2 + k: 4, + relevantRetrieved: 2, + }) + // The degenerate trio must not come back: recall was identical to hitAtK by construction, + // F1 was a re-encoding of precision, and NDCG's ideal set was built from what was found. + for (const gone of ["recallAtK", "f1AtK", "ndcg", "totalRelevant"]) { + expect(metrics).not.toHaveProperty(gone) + } +}) + +test("precision and MRR distinguish rankings that the old NDCG scored identically", async () => { + // Old behaviour: IDCG was built from the retrieved relevant count, so "one relevant at rank 1" + // and "four relevant at ranks 1-4" both scored NDCG 1.0 and recall 100%. + const onlyFirst = await calculateRetrievalMetrics( + judgeReturning( + JSON.stringify([ + { id: "result_1", relevant: 1 }, + { id: "result_2", relevant: 0 }, + { id: "result_3", relevant: 0 }, + { id: "result_4", relevant: 0 }, + ]) + ), + "q", + "gt", + RESULTS + ) + const allFour = await calculateRetrievalMetrics( + judgeReturning( + JSON.stringify([ + { id: "result_1", relevant: 1 }, + { id: "result_2", relevant: 1 }, + { id: "result_3", relevant: 1 }, + { id: "result_4", relevant: 1 }, + ]) + ), + "q", + "gt", + RESULTS + ) + + // Both "hit" and both rank a relevant result first, so those two agree... + expect(onlyFirst!.hitAtK).toBe(allFour!.hitAtK) + expect(onlyFirst!.mrr).toBe(allFour!.mrr) + // ...but precision now separates them, which is the only honest signal available here. + expect(onlyFirst!.precisionAtK).toBe(0.25) + expect(allFour!.precisionAtK).toBe(1) +}) + +test("a judge failure yields no metrics instead of a zero score", async () => { + // Coercing a rate-limit or timeout to relevant:0 was indistinguishable from "retrieved + // nothing useful" and quietly dragged the provider's retrieval numbers down. + const metrics = await calculateRetrievalMetrics(judgeThatFails(), "q", "gt", RESULTS) + + expect(metrics).toBeUndefined() +}) + +test("an unparseable judge response yields no metrics", async () => { + const metrics = await calculateRetrievalMetrics( + judgeReturning("I'm afraid I can't help with that."), + "q", + "gt", + RESULTS + ) + + expect(metrics).toBeUndefined() +}) + +test("retrieving nothing is a real zero, not a missing measurement", async () => { + const metrics = await calculateRetrievalMetrics(judgeReturning("[]"), "q", "gt", []) + + expect(metrics).toEqual({ hitAtK: 0, precisionAtK: 0, mrr: 0, k: 0, relevantRetrieved: 0 }) +}) diff --git a/src/orchestrator/phases/retrieval-eval.ts b/src/orchestrator/phases/retrieval-eval.ts index 344197c..f03744e 100644 --- a/src/orchestrator/phases/retrieval-eval.ts +++ b/src/orchestrator/phases/retrieval-eval.ts @@ -1,6 +1,7 @@ import type { RetrievalMetrics } from "../../types/unified" import type { LanguageModel } from "ai" import { generateText } from "ai" +import { logger } from "../../utils/logger" interface RelevanceResult { id: string @@ -12,7 +13,7 @@ async function evaluateAllChunks( question: string, groundTruth: string, searchResults: unknown[] -): Promise { +): Promise { if (searchResults.length === 0) return [] const formattedResults = searchResults @@ -51,6 +52,10 @@ Where: Return ONLY the JSON array, no other text.` + // A judge that times out, rate-limits, or answers unparseably tells us nothing about + // relevance. Returning all-zeros made that indistinguishable from "the provider retrieved + // nothing useful" and quietly dragged the provider's retrieval numbers down, so failures + // now return null and the question is left out of the aggregate instead. try { const response = await generateText({ model, @@ -59,89 +64,57 @@ Return ONLY the JSON array, no other text.` const jsonMatch = response.text.match(/\[[\s\S]*\]/) if (!jsonMatch) { - return searchResults.map((_, i) => ({ id: `result_${i + 1}`, relevant: 0 as const })) + logger.warn("Relevance judge returned no JSON array; skipping retrieval metrics") + return null } - const parsed = JSON.parse(jsonMatch[0]) as RelevanceResult[] - return parsed - } catch { - return searchResults.map((_, i) => ({ id: `result_${i + 1}`, relevant: 0 as const })) + return JSON.parse(jsonMatch[0]) as RelevanceResult[] + } catch (e) { + logger.warn(`Relevance judge failed; skipping retrieval metrics: ${e}`) + return null } } -function calculateNDCG(relevanceScores: number[], idealRelevant: number): number { - const dcg = relevanceScores.reduce((sum, rel, i) => { - return sum + rel / Math.log2(i + 2) - }, 0) - - const idealScores = Array(relevanceScores.length).fill(0) - for (let i = 0; i < Math.min(idealRelevant, idealScores.length); i++) { - idealScores[i] = 1 - } - const idcg = idealScores.reduce((sum, rel, i) => { - return sum + rel / Math.log2(i + 2) - }, 0) - - return idcg > 0 ? dcg / idcg : 0 -} - +/** + * Retrieval metrics for one question, or `undefined` when the relevance judge could not be + * consulted — the caller leaves those questions out of the aggregate rather than recording + * a zero it cannot justify. + * + * Only metrics that are well defined without a ground-truth relevance count are produced. + * Recall@K, F1@K and NDCG need to know how many relevant memories exist in the corpus; that + * number is not available here, and substituting the retrieved count (as this used to) makes + * recall identical to Hit@K and NDCG blind to anything the provider missed. + */ export async function calculateRetrievalMetrics( model: LanguageModel, question: string, groundTruth: string, searchResults: unknown[], k: number = 10 -): Promise { +): Promise { const resultsToEval = searchResults.slice(0, k) if (resultsToEval.length === 0) { - return { - hitAtK: 0, - precisionAtK: 0, - recallAtK: 0, - f1AtK: 0, - mrr: 0, - ndcg: 0, - k: 0, - relevantRetrieved: 0, - totalRelevant: 1, - } + // Retrieved nothing, so nothing was relevant. This is a real measurement, not a failure. + return { hitAtK: 0, precisionAtK: 0, mrr: 0, k: 0, relevantRetrieved: 0 } } const relevanceResults = await evaluateAllChunks(model, question, groundTruth, resultsToEval) + if (relevanceResults === null) return undefined const relevanceScores = resultsToEval.map((_, i) => { - const id = `result_${i + 1}` - const result = relevanceResults.find((r) => r.id === id) + const result = relevanceResults.find((r) => r.id === `result_${i + 1}`) return result?.relevant === 1 ? 1 : 0 }) const relevantRetrieved = relevanceScores.filter((r) => r === 1).length - const totalRelevant = Math.max(1, relevantRetrieved) - - const hitAtK = relevantRetrieved > 0 ? 1 : 0 - - const precisionAtK = resultsToEval.length > 0 ? relevantRetrieved / resultsToEval.length : 0 - - const recallAtK = relevantRetrieved > 0 ? 1 : 0 - - const f1AtK = - precisionAtK + recallAtK > 0 ? (2 * (precisionAtK * recallAtK)) / (precisionAtK + recallAtK) : 0 - const firstRelevantIndex = relevanceScores.findIndex((r) => r === 1) - const mrr = firstRelevantIndex >= 0 ? 1 / (firstRelevantIndex + 1) : 0 - - const ndcg = calculateNDCG(relevanceScores, totalRelevant) return { - hitAtK, - precisionAtK, - recallAtK, - f1AtK, - mrr, - ndcg, + hitAtK: relevantRetrieved > 0 ? 1 : 0, + precisionAtK: relevantRetrieved / resultsToEval.length, + mrr: firstRelevantIndex >= 0 ? 1 / (firstRelevantIndex + 1) : 0, k: resultsToEval.length, relevantRetrieved, - totalRelevant, } } diff --git a/src/types/unified.ts b/src/types/unified.ts index e4a0deb..52e1494 100644 --- a/src/types/unified.ts +++ b/src/types/unified.ts @@ -30,25 +30,28 @@ export interface UnifiedQuestion { export type SearchResult = unknown +/** + * Retrieval quality over the results a provider returned for one question. + * + * Deliberately limited to metrics that are well defined without knowing how many relevant + * memories exist in the corpus. Recall@K, F1@K and NDCG were previously reported here, but + * with no ground-truth denominator they were computed against the retrieved set itself: recall + * reduced to exactly hitAtK, F1 to a re-encoding of precision, and NDCG's ideal ranking was + * built from what the provider happened to find, so it could never register a miss. See the + * issue trail on #67 before adding them back — they need per-question relevance labels first. + */ export interface RetrievalMetrics { hitAtK: number precisionAtK: number - recallAtK: number - f1AtK: number mrr: number - ndcg: number k: number relevantRetrieved: number - totalRelevant: number } export interface RetrievalAggregates { hitAtK: number precisionAtK: number - recallAtK: number - f1AtK: number mrr: number - ndcg: number k: number } diff --git a/ui/app/compare/[compareId]/page.tsx b/ui/app/compare/[compareId]/page.tsx index bf5e0f8..04fcd05 100644 --- a/ui/app/compare/[compareId]/page.tsx +++ b/ui/app/compare/[compareId]/page.tsx @@ -21,6 +21,18 @@ import { Tooltip } from "@/components/tooltip" const POLL_INTERVAL = 2000 // 2 seconds +/** + * Retrieval columns, defined once so the header and body cannot drift apart. + * + * Recall, F1 and NDCG used to appear here. They were degenerate without ground-truth relevance + * counts — recall was identical to Hit@K by construction — so they are no longer reported (#67). + */ +const RETRIEVAL_COLUMNS = [ + { key: "hitAtK", label: "Hit@K", format: (v: number) => `${(v * 100).toFixed(0)}%` }, + { key: "precisionAtK", label: "Precision", format: (v: number) => `${(v * 100).toFixed(0)}%` }, + { key: "mrr", label: "MRR", format: (v: number) => v.toFixed(2) }, +] as const + export default function CompareDetailPage() { const params = useParams() const router = useRouter() @@ -567,27 +579,17 @@ export default function CompareDetailPage() { - - - - - - - + {RETRIEVAL_COLUMNS.map((c) => ( + + ))} @@ -602,7 +604,7 @@ export default function CompareDetailPage() { if (rows.length === 0) { return ( - @@ -610,20 +612,11 @@ export default function CompareDetailPage() { } // Find best values and FIRST index for each metric - const metrics = [ - "hitAtK", - "precisionAtK", - "recallAtK", - "f1AtK", - "mrr", - "ndcg", - ] as const - const bestByMetric = metrics.reduce( - (acc, metric) => { - const values = rows.map((r) => r.retrieval[metric]) + const bestByMetric = RETRIEVAL_COLUMNS.reduce( + (acc, { key }) => { + const values = rows.map((r) => r.retrieval[key]) const bestValue = Math.max(...values) - const firstBestIndex = values.findIndex((v) => v === bestValue) - acc[metric] = { value: bestValue, firstIndex: firstBestIndex } + acc[key] = { value: bestValue, firstIndex: values.indexOf(bestValue) } return acc }, {} as Record @@ -632,72 +625,19 @@ export default function CompareDetailPage() { return rows.map((row, rowIndex) => ( - - - - - - + {RETRIEVAL_COLUMNS.map(({ key, format }) => ( + + ))} )) })()} diff --git a/ui/components/benchmark-results.tsx b/ui/components/benchmark-results.tsx index 0fb6bb6..52e5d9f 100644 --- a/ui/components/benchmark-results.tsx +++ b/ui/components/benchmark-results.tsx @@ -111,10 +111,7 @@ export interface LatencyStats { export interface RetrievalStats { hitAtK: number precisionAtK: number - recallAtK: number - f1AtK: number mrr: number - ndcg: number k: number } @@ -217,7 +214,7 @@ export function RetrievalMetrics({ retrieval, byQuestionType }: RetrievalMetrics Retrieval Quality (K={retrieval.k}) -
+
Hit@{retrieval.k} @@ -227,24 +224,19 @@ export function RetrievalMetrics({ retrieval, byQuestionType }: RetrievalMetrics
found relevant
-
-
MRR
-
{retrieval.mrr.toFixed(2)}
-
mean reciprocal rank
-
-
-
NDCG
-
{retrieval.ndcg.toFixed(2)}
-
ranking quality
-
- F1@{retrieval.k} + Precision@{retrieval.k}
- {(retrieval.f1AtK * 100).toFixed(0)}% + {(retrieval.precisionAtK * 100).toFixed(0)}%
-
precision-recall balance
+
relevant out of retrieved
+
+
+
MRR
+
{retrieval.mrr.toFixed(2)}
+
mean reciprocal rank
@@ -269,47 +261,37 @@ export function RetrievalMetrics({ retrieval, byQuestionType }: RetrievalMetrics
- {(["hitAtK", "precisionAtK", "recallAtK", "f1AtK", "mrr", "ndcg"] as const).map( - (metric) => { - const labels: Record = { - hitAtK: `Hit@${retrieval.k}`, - precisionAtK: "Precision", - recallAtK: "Recall", - f1AtK: "F1", - mrr: "MRR", - ndcg: "NDCG", - } - const tooltips: Record = { - hitAtK: "found at least one relevant result", - precisionAtK: "relevant results out of retrieved", - recallAtK: "found relevant content", - f1AtK: "precision-recall balance", - mrr: "mean reciprocal rank", - ndcg: "ranking quality score", - } - const isPercentage = ["hitAtK", "precisionAtK", "recallAtK", "f1AtK"].includes( - metric - ) - const format = (v: number) => - isPercentage ? `${(v * 100).toFixed(1)}%` : v.toFixed(3) - - return ( - - - - {questionTypes.map(([type, stats]) => ( - - ))} - - ) + {(["hitAtK", "precisionAtK", "mrr"] as const).map((metric) => { + const labels: Record = { + hitAtK: `Hit@${retrieval.k}`, + precisionAtK: "Precision", + mrr: "MRR", } - )} + const tooltips: Record = { + hitAtK: "found at least one relevant result", + precisionAtK: "relevant results out of retrieved", + mrr: "mean reciprocal rank", + } + const isPercentage = ["hitAtK", "precisionAtK"].includes(metric) + const format = (v: number) => + isPercentage ? `${(v * 100).toFixed(1)}%` : v.toFixed(3) + + return ( + + + + {questionTypes.map(([type, stats]) => ( + + ))} + + ) + })}
+ Provider - Hit@K - - Precision - - Recall - - F1 - - MRR - - NDCG - + {c.label} +
+ Retrieval metrics not available
{row.provider} - - {(row.retrieval.hitAtK * 100).toFixed(0)}% - - - - {(row.retrieval.precisionAtK * 100).toFixed(0)}% - - - - {(row.retrieval.recallAtK * 100).toFixed(0)}% - - - - {(row.retrieval.f1AtK * 100).toFixed(0)}% - - - - {row.retrieval.mrr.toFixed(2)} - - - - {row.retrieval.ndcg.toFixed(2)} - - + + {format(row.retrieval[key])} + +
- {labels[metric]} - - {format(retrieval[metric])} - - {stats.retrieval ? format(stats.retrieval[metric]) : "—"} -
+ {labels[metric]} + + {format(retrieval[metric])} + + {stats.retrieval ? format(stats.retrieval[metric]) : "—"} +
diff --git a/ui/lib/api.ts b/ui/lib/api.ts index 8a22190..5c97b50 100644 --- a/ui/lib/api.ts +++ b/ui/lib/api.ts @@ -402,10 +402,7 @@ export interface BenchmarkResult { retrieval?: { hitAtK: number precisionAtK: number - recallAtK: number - f1AtK: number mrr: number - ndcg: number k: number } evaluations?: EvaluationResult[]