diff --git a/.gitignore b/.gitignore index 3c3273de2a..6446eb7908 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,7 @@ temp # AI .claude .agent + +# Eval results +evals/results/ +evals/report.json diff --git a/charts/line.md b/charts/line.md index 7f251661fb..f31f0393c8 100644 --- a/charts/line.md +++ b/charts/line.md @@ -39,7 +39,6 @@ const data = [ const chart = new Chart({ container: 'container', autoFit: true, - height: 360, }); chart.options({ diff --git a/evals/.env.example b/evals/.env.example new file mode 100644 index 0000000000..028bbd80ca --- /dev/null +++ b/evals/.env.example @@ -0,0 +1,6 @@ +# LLM 配置(OpenAI 兼容接口,默认 Moonshot) +# 复制为 evals/.env 并填入真实 key,或直接 export 环境变量 +LLM_BASE_URL=https://api.moonshot.cn/v1 +LLM_API_KEY=sk-your-key-here +LLM_MODEL=moonshot-v1-8k +LLM_TEMPERATURE=0 diff --git a/evals/cases/line.jsonl b/evals/cases/line.jsonl new file mode 100644 index 0000000000..57336b7589 --- /dev/null +++ b/evals/cases/line.jsonl @@ -0,0 +1,20 @@ +{"id": "line-trend-basic", "prompt": "我在做一个后台管理系统,需要展示过去 30 天每天的活跃用户数,用折线图看趋势变化", "groundTruth": "使用 type: 'line';x 轴为日期字段(如 date),y 轴为活跃用户数(如 value 或 dau);数据内联 30 条左右;单系列无 color 通道"} +{"id": "line-trend-multi", "prompt": "做一个销售看板,对比华东、华南、华北三个区域过去 12 个月的销售额走势", "groundTruth": "使用 type: 'line';encode 必须包含 color 通道映射区域字段;数据包含三个区域各 12 个月;x 轴为月份"} +{"id": "line-smooth-temp", "prompt": "展示一天 24 小时的温度变化,平滑一点的曲线看起来更舒服", "groundTruth": "encode 中设置 shape: 'smooth';x 轴为小时(0-23 或 HH:00 格式),y 轴为温度值"} +{"id": "line-step-price", "prompt": "商品价格每隔一段时间会调整一次,用阶梯图展示价格变动历史", "groundTruth": "encode 中设置 shape: 'hv' / 'vh' / 'hvh' 之一;数据体现价格跳变(同一价格持续多期后突变)"} +{"id": "line-label-point", "prompt": "周报需要展示本周每天的订单量,要求每个数据点都标出具体数值", "groundTruth": "包含 labels 配置,text 指向数值字段;position 为 'top' 或 'outside';可搭配 point mark 突出数据点"} +{"id": "line-point-combo", "prompt": "既要趋势线又要清晰标出每个采样点,比如传感器每小时的读数", "groundTruth": "使用 type: 'view',children 同时包含 type: 'line' 和 type: 'point';两者 encode 一致"} +{"id": "line-custom-color", "prompt": "品牌色是 #FF6B35,画一个用品牌色的折线图,线宽要粗一点", "groundTruth": "style 中设置 stroke: '#FF6B35' 和 lineWidth >= 2;单系列场景下通过 style 固定颜色"} +{"id": "line-dashed-compare", "prompt": "实际值用实线,预测值用虚线,画在一个图上对比", "groundTruth": "通过 color 通道区分实际/预测系列,并在 style 或 scale 中设置 lineDash;或分两条 line mark 分别设置 lineDash"} +{"id": "line-ref-line", "prompt": "画一个 KPI 达成趋势图,加一条目标线 90%,看看哪些月份没达标", "groundTruth": "使用 line mark + 额外的参考线(可通过 annotations 或额外 data + transform 实现);参考线值为 90"} +{"id": "line-tooltip-format", "prompt": "tooltip 里显示「万元」单位,数值保留两位小数", "groundTruth": "tooltip.items 中设置 valueFormatter,包含 '万元' 字样和 toFixed(2) 或类似格式化"} +{"id": "line-dark-theme", "prompt": "大屏展示用,暗色主题的折线图", "groundTruth": "设置 theme: 'classicDark' 或 'dark';背景、文字颜色适配暗色"} +{"id": "line-title-subtitle", "prompt": "图表要有标题「月度营收趋势」和副标题「数据截至 2026-09」,左对齐", "groundTruth": "title 为对象,包含 title 和 subtitle 字段;align 为 'left'"} +{"id": "line-null-data", "prompt": "有些日期没有数据,折线在这些地方断开而不是连起来", "groundTruth": "数据中含 null/undefined 值或缺失字段;line mark 默认断开或显式配置 connectNulls: false"} +{"id": "line-log-axis", "prompt": "数据跨度太大(从 100 到 100000),y 轴用对数刻度", "groundTruth": "scale.y 中设置 type: 'log' 或 transform: 'log'"} +{"id": "line-area-fill", "prompt": "趋势线下面填充淡淡的颜色,增强视觉效果", "groundTruth": "使用 type: 'area' 或 line + area 组合;设置 fillOpacity 或 style.fill 实现渐变/填充"} +{"id": "line-legend-top", "prompt": "图例放到图表顶部,节省右侧空间", "groundTruth": "legend.color 或 legend 中设置 position: 'top'"} +{"id": "line-axis-title", "prompt": "x 轴标题写「日期」,y 轴标题写「销售额(万元)」", "groundTruth": "axis.x.title 和 axis.y.title 分别设置为对应文字"} +{"id": "line-time-format", "prompt": "x 轴日期显示为「MM-DD」格式,不要年份", "groundTruth": "scale.x 或 axis.x 中设置 labelFormatter 或 tickFormat,只显示月日"} +{"id": "line-large-data", "prompt": "有 365 天的数据,直接画太密了,帮忙抽稀显示", "groundTruth": "使用 transform 中的 sample 或 filter;或数据预处理聚合(如周平均)"} +{"id": "line-dual-axis", "prompt": "销售额和订单量量级差太多,用双 y 轴展示", "groundTruth": "使用 type: 'view',children 包含两个 line mark,第二个设置 scale: { y: { independent: true } } 或 axis: { y: { position: 'right' } }"} diff --git a/evals/generate.mjs b/evals/generate.mjs new file mode 100644 index 0000000000..b80919f277 --- /dev/null +++ b/evals/generate.mjs @@ -0,0 +1,57 @@ +// Code generation: simulate an AI Coding Agent handling user queries. +// The agent activates the g2plot-v3 skill, reads referenced files via tools, +// and writes generated code to evals/results/. +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { config as loadEnv } from 'dotenv'; +import { createModel } from './lib/llm.mjs'; +import { createGenerateAgent } from './lib/generate-agent.mjs'; +import { loadCases } from './lib/load-cases.mjs'; +import { RESULTS_DIR, EVALS_DIR } from './lib/const.mjs'; + +loadEnv({ path: path.join(EVALS_DIR, '.env'), quiet: true }); + +const extractCode = (text) => + (text.match(/```(?:ts|typescript)\n([\s\S]*?)```/)?.[1] ?? text).trim(); + +async function main() { + const { model, temperature } = createModel(); + const agent = createGenerateAgent({ model, temperature }); + + const cases = await loadCases(); + console.log(`Generating code for ${cases.length} case(s) using ${model.modelId}...`); + + await fs.rm(RESULTS_DIR, { recursive: true, force: true }); + await fs.mkdir(RESULTS_DIR, { recursive: true }); + + const results = []; + for (const evalCase of cases) { + process.stdout.write(` ${evalCase.id} ... `); + let code = ''; + let steps = 0; + try { + const result = await agent.generate({ prompt: evalCase.prompt }); + steps = result.steps?.length ?? 0; + code = extractCode(result.text); + await fs.writeFile(path.join(RESULTS_DIR, `${evalCase.id}.ts`), code); + console.log(`ok (${steps} steps)`); + } catch (err) { + console.log(`error: ${err.message?.slice(0, 80)}`); + } + results.push({ id: evalCase.id, chart: evalCase.chart, code, steps }); + } + + await fs.writeFile( + path.join(RESULTS_DIR, 'eval-result.json'), + JSON.stringify(results, null, 2) + ); + + const ok = results.filter((r) => r.code).length; + console.log(`\nGenerated ${ok}/${results.length} case(s) with code.`); + if (ok < results.length) process.exitCode = 1; +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/evals/judge.mjs b/evals/judge.mjs new file mode 100644 index 0000000000..111726afa6 --- /dev/null +++ b/evals/judge.mjs @@ -0,0 +1,105 @@ +// Judge: render each generated case in a headless browser via G2 CDN, +// take a screenshot, and write score back to eval-result.json. +// score: 100 = rendered without errors, 0 = runtime error or empty code. +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import puppeteer from 'puppeteer'; +import { build } from 'esbuild'; +import { RESULTS_DIR } from './lib/const.mjs'; + +const RESULT_FILE = path.join(RESULTS_DIR, 'eval-result.json'); +const G2_CDN = 'https://unpkg.com/@antv/g2@5/dist/g2.min.js'; + +// Shim @antv/g2 imports to the browser global window.G2 loaded from CDN. +const g2ShimPlugin = { + name: 'g2-shim', + setup(b) { + b.onResolve({ filter: /^@antv\/g2$/ }, () => ({ path: '@antv/g2', namespace: 'g2-shim' })); + b.onLoad({ filter: /.*/, namespace: 'g2-shim' }, () => ({ + contents: 'module.exports = window.G2;', + loader: 'js', + })); + }, +}; + +async function transpile(code) { + const result = await build({ + stdin: { contents: code, loader: 'ts', resolveDir: process.cwd() }, + bundle: true, + format: 'iife', + target: 'es2020', + write: false, + plugins: [g2ShimPlugin], + }); + return result.outputFiles[0].text; +} + +async function buildHtml(code) { + const js = await transpile(code); + return ` + +
+ + + + +`; +} + +async function judgeCase(browser, item) { + const screenshot = path.join(RESULTS_DIR, `${item.id}.png`); + if (!item.code) { + return { ...item, score: 0, error: 'empty code' }; + } + const page = await browser.newPage(); + try { + await page.setViewport({ width: 820, height: 420 }); + await page.setContent(await buildHtml(item.code), { waitUntil: 'networkidle0', timeout: 30000 }); + await page.waitForSelector('#container canvas', { timeout: 10000 }).catch(() => {}); + const errors = await page.evaluate(() => window.__errors); + await page.screenshot({ path: screenshot }); + if (errors && errors.length > 0) { + return { ...item, score: 0, error: errors.join('; ').slice(0, 500) }; + } + return { ...item, score: 100, screenshot: `${item.id}.png` }; + } catch (err) { + return { ...item, score: 0, error: String(err.message || err).slice(0, 500) }; + } finally { + await page.close(); + } +} + +async function main() { + const items = JSON.parse(await fs.readFile(RESULT_FILE, 'utf-8')); + console.log(`Judging ${items.length} case(s) in headless browser...`); + + const browser = await puppeteer.launch({ headless: 'new' }); + const judged = []; + for (const item of items) { + process.stdout.write(` ${item.id} ... `); + const result = await judgeCase(browser, item); + judged.push(result); + console.log(result.score === 100 ? 'pass' : `fail: ${(result.error || '').slice(0, 80)}`); + } + await browser.close(); + + await fs.writeFile(RESULT_FILE, JSON.stringify(judged, null, 2)); + + const passed = judged.filter((r) => r.score === 100).length; + console.log(`\nJudged ${passed}/${judged.length} passed. Screenshots saved to evals/results/.`); + if (passed < judged.length) process.exitCode = 1; +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/evals/lib/const.mjs b/evals/lib/const.mjs new file mode 100644 index 0000000000..65cdfa0d8c --- /dev/null +++ b/evals/lib/const.mjs @@ -0,0 +1,8 @@ +// Shared path constants for eval scripts. +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export const EVALS_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +export const ROOT = path.resolve(EVALS_DIR, '..'); +export const RESULTS_DIR = path.join(EVALS_DIR, 'results'); +export const CASES_DIR = path.join(EVALS_DIR, 'cases'); diff --git a/evals/lib/generate-agent.mjs b/evals/lib/generate-agent.mjs new file mode 100644 index 0000000000..4806c2732e --- /dev/null +++ b/evals/lib/generate-agent.mjs @@ -0,0 +1,33 @@ +// Build a nano AI Coding Agent for G2Plot v3 eval. +import { ToolLoopAgent, stepCountIs } from 'ai'; +import { activeSkill } from './tools/active-skill.mjs'; +import { readFile } from './tools/read-file.mjs'; +import { curl } from './tools/curl.mjs'; + +const INSTRUCTIONS = `You are an AI Coding Agent that generates runnable G2 v5 chart TypeScript code from user requirements. + +## Workflow + +1. Call activeSkill to activate the g2plot-v3 skill and get its content. +2. Use the selection guide in the skill to pick a chart type, then readFile the referenced \`charts/