feat: add evals = generate + judge for v3

.gitignore

@@ -60,3 +60,7 @@ temp
 # AI
 .claude
 .agent
+
+# Eval results
+evals/results/
+evals/report.json

charts/line.md

@@ -39,7 +39,6 @@ const data = [
 const chart = new Chart({
   container: 'container',
   autoFit: true,
-  height: 360,
 });
 
 chart.options({

evals/.env.example

@@ -0,0 +1,6 @@
+# LLM 配置（OpenAI 兼容接口，默认 Moonshot）
+# 复制为 evals/.env 并填入真实 key，或直接 export 环境变量
+LLM_BASE_URL=https://api.moonshot.cn/v1
+LLM_API_KEY=sk-your-key-here
+LLM_MODEL=moonshot-v1-8k
+LLM_TEMPERATURE=0

evals/cases/line.jsonl

@@ -0,0 +1,20 @@
+{"id": "line-trend-basic", "prompt": "我在做一个后台管理系统，需要展示过去 30 天每天的活跃用户数，用折线图看趋势变化", "groundTruth": "使用 type: 'line'；x 轴为日期字段（如 date），y 轴为活跃用户数（如 value 或 dau）；数据内联 30 条左右；单系列无 color 通道"}
+{"id": "line-trend-multi", "prompt": "做一个销售看板，对比华东、华南、华北三个区域过去 12 个月的销售额走势", "groundTruth": "使用 type: 'line'；encode 必须包含 color 通道映射区域字段；数据包含三个区域各 12 个月；x 轴为月份"}
+{"id": "line-smooth-temp", "prompt": "展示一天 24 小时的温度变化，平滑一点的曲线看起来更舒服", "groundTruth": "encode 中设置 shape: 'smooth'；x 轴为小时（0-23 或 HH:00 格式），y 轴为温度值"}
+{"id": "line-step-price", "prompt": "商品价格每隔一段时间会调整一次，用阶梯图展示价格变动历史", "groundTruth": "encode 中设置 shape: 'hv' / 'vh' / 'hvh' 之一；数据体现价格跳变（同一价格持续多期后突变）"}
+{"id": "line-label-point", "prompt": "周报需要展示本周每天的订单量，要求每个数据点都标出具体数值", "groundTruth": "包含 labels 配置，text 指向数值字段；position 为 'top' 或 'outside'；可搭配 point mark 突出数据点"}
+{"id": "line-point-combo", "prompt": "既要趋势线又要清晰标出每个采样点，比如传感器每小时的读数", "groundTruth": "使用 type: 'view'，children 同时包含 type: 'line' 和 type: 'point'；两者 encode 一致"}
+{"id": "line-custom-color", "prompt": "品牌色是 #FF6B35，画一个用品牌色的折线图，线宽要粗一点", "groundTruth": "style 中设置 stroke: '#FF6B35' 和 lineWidth >= 2；单系列场景下通过 style 固定颜色"}
+{"id": "line-dashed-compare", "prompt": "实际值用实线，预测值用虚线，画在一个图上对比", "groundTruth": "通过 color 通道区分实际/预测系列，并在 style 或 scale 中设置 lineDash；或分两条 line mark 分别设置 lineDash"}
+{"id": "line-ref-line", "prompt": "画一个 KPI 达成趋势图，加一条目标线 90%，看看哪些月份没达标", "groundTruth": "使用 line mark + 额外的参考线（可通过 annotations 或额外 data + transform 实现）；参考线值为 90"}
+{"id": "line-tooltip-format", "prompt": "tooltip 里显示「万元」单位，数值保留两位小数", "groundTruth": "tooltip.items 中设置 valueFormatter，包含 '万元' 字样和 toFixed(2) 或类似格式化"}
+{"id": "line-dark-theme", "prompt": "大屏展示用，暗色主题的折线图", "groundTruth": "设置 theme: 'classicDark' 或 'dark'；背景、文字颜色适配暗色"}
+{"id": "line-title-subtitle", "prompt": "图表要有标题「月度营收趋势」和副标题「数据截至 2026-09」，左对齐", "groundTruth": "title 为对象，包含 title 和 subtitle 字段；align 为 'left'"}
+{"id": "line-null-data", "prompt": "有些日期没有数据，折线在这些地方断开而不是连起来", "groundTruth": "数据中含 null/undefined 值或缺失字段；line mark 默认断开或显式配置 connectNulls: false"}
+{"id": "line-log-axis", "prompt": "数据跨度太大（从 100 到 100000），y 轴用对数刻度", "groundTruth": "scale.y 中设置 type: 'log' 或 transform: 'log'"}
+{"id": "line-area-fill", "prompt": "趋势线下面填充淡淡的颜色，增强视觉效果", "groundTruth": "使用 type: 'area' 或 line + area 组合；设置 fillOpacity 或 style.fill 实现渐变/填充"}
+{"id": "line-legend-top", "prompt": "图例放到图表顶部，节省右侧空间", "groundTruth": "legend.color 或 legend 中设置 position: 'top'"}
+{"id": "line-axis-title", "prompt": "x 轴标题写「日期」，y 轴标题写「销售额（万元）」", "groundTruth": "axis.x.title 和 axis.y.title 分别设置为对应文字"}
+{"id": "line-time-format", "prompt": "x 轴日期显示为「MM-DD」格式，不要年份", "groundTruth": "scale.x 或 axis.x 中设置 labelFormatter 或 tickFormat，只显示月日"}
+{"id": "line-large-data", "prompt": "有 365 天的数据，直接画太密了，帮忙抽稀显示", "groundTruth": "使用 transform 中的 sample 或 filter；或数据预处理聚合（如周平均）"}
+{"id": "line-dual-axis", "prompt": "销售额和订单量量级差太多，用双 y 轴展示", "groundTruth": "使用 type: 'view'，children 包含两个 line mark，第二个设置 scale: { y: { independent: true } } 或 axis: { y: { position: 'right' } }"}

evals/generate.mjs

@@ -0,0 +1,57 @@
+// Code generation: simulate an AI Coding Agent handling user queries.
+// The agent activates the g2plot-v3 skill, reads referenced files via tools,
+// and writes generated code to evals/results/.
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { config as loadEnv } from 'dotenv';
+import { createModel } from './lib/llm.mjs';
+import { createGenerateAgent } from './lib/generate-agent.mjs';
+import { loadCases } from './lib/load-cases.mjs';
+import { RESULTS_DIR, EVALS_DIR } from './lib/const.mjs';
+
+loadEnv({ path: path.join(EVALS_DIR, '.env'), quiet: true });
+
+const extractCode = (text) =>
+  (text.match(/```(?:ts|typescript)\n([\s\S]*?)```/)?.[1] ?? text).trim();
+
+async function main() {
+  const { model, temperature } = createModel();
+  const agent = createGenerateAgent({ model, temperature });
+
+  const cases = await loadCases();
+  console.log(`Generating code for ${cases.length} case(s) using ${model.modelId}...`);
+
+  await fs.rm(RESULTS_DIR, { recursive: true, force: true });
+  await fs.mkdir(RESULTS_DIR, { recursive: true });
+
+  const results = [];
+  for (const evalCase of cases) {
+    process.stdout.write(`  ${evalCase.id} ... `);
+    let code = '';
+    let steps = 0;
+    try {
+      const result = await agent.generate({ prompt: evalCase.prompt });
+      steps = result.steps?.length ?? 0;
+      code = extractCode(result.text);
+      await fs.writeFile(path.join(RESULTS_DIR, `${evalCase.id}.ts`), code);
+      console.log(`ok (${steps} steps)`);
+    } catch (err) {
+      console.log(`error: ${err.message?.slice(0, 80)}`);
+    }
+    results.push({ id: evalCase.id, chart: evalCase.chart, code, steps });
+  }
+
+  await fs.writeFile(
+    path.join(RESULTS_DIR, 'eval-result.json'),
+    JSON.stringify(results, null, 2)
+  );
+
+  const ok = results.filter((r) => r.code).length;
+  console.log(`\nGenerated ${ok}/${results.length} case(s) with code.`);
+  if (ok < results.length) process.exitCode = 1;
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

evals/judge.mjs

@@ -0,0 +1,105 @@
+// Judge: render each generated case in a headless browser via G2 CDN,
+// take a screenshot, and write score back to eval-result.json.
+// score: 100 = rendered without errors, 0 = runtime error or empty code.
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import puppeteer from 'puppeteer';
+import { build } from 'esbuild';
+import { RESULTS_DIR } from './lib/const.mjs';
+
+const RESULT_FILE = path.join(RESULTS_DIR, 'eval-result.json');
+const G2_CDN = 'https://unpkg.com/@antv/g2@5/dist/g2.min.js';
+
+// Shim @antv/g2 imports to the browser global window.G2 loaded from CDN.
+const g2ShimPlugin = {
+  name: 'g2-shim',
+  setup(b) {
+    b.onResolve({ filter: /^@antv\/g2$/ }, () => ({ path: '@antv/g2', namespace: 'g2-shim' }));
+    b.onLoad({ filter: /.*/, namespace: 'g2-shim' }, () => ({
+      contents: 'module.exports = window.G2;',
+      loader: 'js',
+    }));
+  },
+};
+
+async function transpile(code) {
+  const result = await build({
+    stdin: { contents: code, loader: 'ts', resolveDir: process.cwd() },
+    bundle: true,
+    format: 'iife',
+    target: 'es2020',
+    write: false,
+    plugins: [g2ShimPlugin],
+  });
+  return result.outputFiles[0].text;
+}
+
+async function buildHtml(code) {
+  const js = await transpile(code);
+  return `<!DOCTYPE html>
+<html>
+<head><meta charset="utf-8"><script src="${G2_CDN}"></script></head>
+<body style="margin:0;background:#fff;">
+<div id="container" style="width:800px;height:400px;"></div>
+<script>
+window.__errors = [];
+window.addEventListener('error', (e) => window.__errors.push(e.message));
+window.addEventListener('unhandledrejection', (e) => window.__errors.push(String(e.reason)));
+try {
+  ${js}
+} catch (err) {
+  window.__errors.push(String(err && err.stack || err));
+}
+</script>
+</body>
+</html>`;
+}
+
+async function judgeCase(browser, item) {
+  const screenshot = path.join(RESULTS_DIR, `${item.id}.png`);
+  if (!item.code) {
+    return { ...item, score: 0, error: 'empty code' };
+  }
+  const page = await browser.newPage();
+  try {
+    await page.setViewport({ width: 820, height: 420 });
+    await page.setContent(await buildHtml(item.code), { waitUntil: 'networkidle0', timeout: 30000 });
+    await page.waitForSelector('#container canvas', { timeout: 10000 }).catch(() => {});
+    const errors = await page.evaluate(() => window.__errors);
+    await page.screenshot({ path: screenshot });
+    if (errors && errors.length > 0) {
+      return { ...item, score: 0, error: errors.join('; ').slice(0, 500) };
+    }
+    return { ...item, score: 100, screenshot: `${item.id}.png` };
+  } catch (err) {
+    return { ...item, score: 0, error: String(err.message || err).slice(0, 500) };
+  } finally {
+    await page.close();
+  }
+}
+
+async function main() {
+  const items = JSON.parse(await fs.readFile(RESULT_FILE, 'utf-8'));
+  console.log(`Judging ${items.length} case(s) in headless browser...`);
+
+  const browser = await puppeteer.launch({ headless: 'new' });
+  const judged = [];
+  for (const item of items) {
+    process.stdout.write(`  ${item.id} ... `);
+    const result = await judgeCase(browser, item);
+    judged.push(result);
+    console.log(result.score === 100 ? 'pass' : `fail: ${(result.error || '').slice(0, 80)}`);
+  }
+  await browser.close();
+
+  await fs.writeFile(RESULT_FILE, JSON.stringify(judged, null, 2));
+
+  const passed = judged.filter((r) => r.score === 100).length;
+  console.log(`\nJudged ${passed}/${judged.length} passed. Screenshots saved to evals/results/.`);
+  if (passed < judged.length) process.exitCode = 1;
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

evals/lib/const.mjs

@@ -0,0 +1,8 @@
+// Shared path constants for eval scripts.
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+export const EVALS_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+export const ROOT = path.resolve(EVALS_DIR, '..');
+export const RESULTS_DIR = path.join(EVALS_DIR, 'results');
+export const CASES_DIR = path.join(EVALS_DIR, 'cases');

evals/lib/generate-agent.mjs

@@ -0,0 +1,33 @@
+// Build a nano AI Coding Agent for G2Plot v3 eval.
+import { ToolLoopAgent, stepCountIs } from 'ai';
+import { activeSkill } from './tools/active-skill.mjs';
+import { readFile } from './tools/read-file.mjs';
+import { curl } from './tools/curl.mjs';
+
+const INSTRUCTIONS = `You are an AI Coding Agent that generates runnable G2 v5 chart TypeScript code from user requirements.
+
+## Workflow
+
+1. Call activeSkill to activate the g2plot-v3 skill and get its content.
+2. Use the selection guide in the skill to pick a chart type, then readFile the referenced \`charts/<chart>.md\` component prompt.
+3. readFile any referenced files under references/ or examples/ as needed (data patterns, API cheatsheet, runnable examples).
+4. If local docs cannot cover a G2 v5 API, use curl to call the AntV context retrieval service mentioned in the skill.
+5. Generate the final code.
+
+## Hard requirements
+
+- Output exactly one \`\`\`ts code block in the final answer, with no explanatory text outside it.
+- The code must use @antv/g2 v5 (import { Chart } from '@antv/g2'). Do not use any G2 v4 or G2Plot v2 APIs.
+- Data must be self-contained and inlined in the code (const data = [...]) with semantic field names.
+- Create the chart with new Chart({ container: 'container', autoFit: true, height: 360 }) and end with chart.render().`;
+
+export function createGenerateAgent({ model, temperature }) {
+  return new ToolLoopAgent({
+    model,
+    instructions: INSTRUCTIONS,
+    temperature,
+    tools: { activeSkill, readFile, curl },
+    // Max steps: activeSkill -> read charts/<chart>.md -> read examples -> generate code.
+    stopWhen: stepCountIs(30),
+  });
+}

evals/lib/llm.mjs

@@ -0,0 +1,26 @@
+// OpenAI-compatible LLM client (defaults to Moonshot).
+// Config is read from env: LLM_BASE_URL / LLM_API_KEY / LLM_MODEL / LLM_TEMPERATURE.
+import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
+
+export function createModel() {
+  const baseURL = process.env.LLM_BASE_URL || 'https://api.moonshot.cn/v1';
+  const apiKey = process.env.LLM_API_KEY;
+  const modelId = process.env.LLM_MODEL || 'moonshot-v1-8k';
+
+  if (!apiKey) {
+    throw new Error(
+      'Missing LLM_API_KEY. Set the environment variable or create evals/.env (see evals/.env.example).'
+    );
+  }
+
+  const provider = createOpenAICompatible({
+    name: 'eval-llm',
+    baseURL,
+    apiKey,
+  });
+
+  return {
+    model: provider(modelId),
+    temperature: Number(process.env.LLM_TEMPERATURE ?? 0),
+  };
+}

evals/lib/load-cases.mjs

@@ -0,0 +1,18 @@
+// Scan evals/cases/<chart>.jsonl and load all evaluation cases.
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { CASES_DIR } from './const.mjs';
+
+export async function loadCases() {
+  const cases = [];
+  const entries = await fs.readdir(CASES_DIR, { withFileTypes: true });
+  for (const entry of entries) {
+    if (!entry.isFile() || !entry.name.endsWith('.jsonl')) continue;
+    const chart = path.basename(entry.name, '.jsonl');
+    const content = await fs.readFile(path.join(CASES_DIR, entry.name), 'utf-8');
+    for (const line of content.split('\n')) {
+      if (line.trim()) cases.push({ chart, ...JSON.parse(line) });
+    }
+  }
+  return cases;
+}

evals/lib/tools/active-skill.mjs

@@ -0,0 +1,36 @@
+// Activate a skill and return its entry content.
+import { tool } from 'ai';
+import { z } from 'zod';
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { ROOT } from '../const.mjs';
+
+const SKILLS = [
+  {
+    name: 'g2plot-v3',
+    description: 'AntV G2Plot v3 可视化组件库，提示词即组件。当用户需要创建、绘制 AntV G2 图表（折线图、柱状图、饼图、面积图、散点图、双轴图、雷达图、热力图、漏斗图等）时使用本 skill。提供 G2 v5 出码规范、图表选型决策、可运行案例与 v4→v5 防幻觉对照。',
+    entry: 'skills/g2plot-v3/SKILL.md',
+  },
+];
+
+const skillList = SKILLS.map((s) => `- ${s.name}: ${s.description}`).join('\n');
+const skillNames = SKILLS.map((s) => s.name).join(', ');
+
+export const activeSkill = tool({
+  description: `Activate a skill and return its content. Available skills:\n${skillList}`,
+  inputSchema: z.object({
+    name: z.string().describe(`Skill name, available: ${skillNames}`),
+  }),
+  execute: async ({ name }) => {
+    const skill = SKILLS.find((s) => s.name === name);
+    if (!skill) {
+      return `Skill "${name}" not found. Available skills: ${skillNames}`;
+    }
+    try {
+      const content = await fs.readFile(path.join(ROOT, skill.entry), 'utf-8');
+      return `[Skill: ${skill.name}]\n${content}`;
+    } catch {
+      return `Error: skill entry file not found: ${skill.entry}`;
+    }
+  },
+});

evals/lib/tools/curl.mjs

@@ -0,0 +1,21 @@
+// Send an HTTP request and return the response body.
+import { tool } from 'ai';
+import { z } from 'zod';
+
+export const curl = tool({
+  description:
+    'Send an HTTP request and return the response body. Use it for the AntV context retrieval service ' +
+    '(GET https://sive.antv.antgroup.com/api/v1/context/retrieve?query=...&library=g2) when local docs cannot cover a G2 v5 API.',
+  inputSchema: z.object({
+    url: z.string().describe('Full URL including query string'),
+    method: z.enum(['GET', 'POST']).default('GET'),
+  }),
+  execute: async ({ url, method }) => {
+    try {
+      const res = await fetch(url, { method });
+      return await res.text();
+    } catch (err) {
+      return `Error: request failed: ${err.message}`;
+    }
+  },
+});

evals/lib/tools/read-file.mjs

@@ -0,0 +1,22 @@
+// Read a file in the repository.
+import { tool } from 'ai';
+import { z } from 'zod';
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { ROOT } from '../const.mjs';
+
+export const readFile = tool({
+  description:
+    'Read a file in the repository, e.g. charts/line.md, references/g2-v5-cheatsheet.md, examples/line/basic.ts. ' +
+    'Use paths referenced by the skill entry file.',
+  inputSchema: z.object({
+    path: z.string().describe('File path relative to the repo root'),
+  }),
+  execute: async ({ path: relPath }) => {
+    try {
+      return await fs.readFile(path.join(ROOT, relPath), 'utf-8');
+    } catch {
+      return `Error: file not found: ${relPath}`;
+    }
+  },
+});

evals/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "@antv/g2plot-evals",
+  "private": true,
+  "type": "module",
+  "description": "G2Plot v3 skill evaluation harness (LLM generate + judge)",
+  "scripts": {
+    "generate": "node generate.mjs",
+    "judge": "node judge.mjs",
+    "eval": "node generate.mjs && node judge.mjs"
+  },
+  "dependencies": {
+    "@ai-sdk/openai-compatible": "^3.0.53",
+    "ai": "^7.0.107",
+    "dotenv": "^18.0.1",
+    "esbuild": "^0.28.2",
+    "puppeteer": "^25.11.0",
+    "zod": "^4.6.5"
+  },
+  "repository": "git@github.com:antvis/G2Plot.git"
+}